1532 lines
56 KiB
Rust
1532 lines
56 KiB
Rust
//! Bounded listeners. HTTP/2 is negotiated through TLS ALPN. Cleartext HTTP/2
|
|
//! is opt-in per listener and accepts both the HTTP/2 prior-knowledge preface and
|
|
//! a validated HTTP/1.1 `Upgrade: h2c` bootstrap.
|
|
use crate::{metrics::Metrics, runtime::App, settings::Http2Connections};
|
|
use anyhow::{Context, Result};
|
|
use axum::{body::Body, Router};
|
|
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
|
|
use bytes::{Buf, Bytes, BytesMut};
|
|
use hyper::{
|
|
body::Incoming,
|
|
server::conn::{http1, http2},
|
|
service::service_fn,
|
|
Request, Response,
|
|
};
|
|
use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer};
|
|
use http_body::Body as HttpBody;
|
|
use std::{
|
|
convert::Infallible,
|
|
io,
|
|
net::SocketAddr,
|
|
pin::Pin,
|
|
sync::{
|
|
atomic::{AtomicU64, AtomicUsize, Ordering},
|
|
Arc,
|
|
},
|
|
task::{Context as TaskContext, Poll},
|
|
time::Duration,
|
|
};
|
|
use tokio::{
|
|
io::{AsyncRead, AsyncWrite, ReadBuf},
|
|
net::TcpListener,
|
|
sync::{Notify, OwnedSemaphorePermit, Semaphore},
|
|
task::JoinSet,
|
|
};
|
|
use tokio_util::sync::CancellationToken;
|
|
use tower::ServiceExt;
|
|
|
|
|
|
const HTTP2_PRIOR_KNOWLEDGE_PREFACE: &[u8; 24] = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n";
|
|
const H2_FRAME_HEADER_LEN: usize = 9;
|
|
|
|
fn header_has_token(headers: &http::HeaderMap, name: &str, token: &str) -> bool {
|
|
headers
|
|
.get_all(name)
|
|
.iter()
|
|
.filter_map(|value| value.to_str().ok())
|
|
.flat_map(|value| value.split(','))
|
|
.any(|value| value.trim().eq_ignore_ascii_case(token))
|
|
}
|
|
|
|
fn single_header<'a>(headers: &'a http::HeaderMap, name: &str) -> std::result::Result<&'a str, &'static str> {
|
|
let mut values = headers.get_all(name).iter();
|
|
let value = values.next().ok_or("missing required h2c header")?;
|
|
if values.next().is_some() {
|
|
return Err("duplicate h2c header");
|
|
}
|
|
value.to_str().map_err(|_| "invalid h2c header encoding")
|
|
}
|
|
|
|
fn h2c_attempt<B>(request: &Request<B>) -> bool {
|
|
request
|
|
.headers()
|
|
.get_all(http::header::UPGRADE)
|
|
.iter()
|
|
.filter_map(|value| value.to_str().ok())
|
|
.flat_map(|value| value.split(','))
|
|
.any(|value| value.trim().eq_ignore_ascii_case("h2c"))
|
|
|| request.headers().contains_key("http2-settings")
|
|
}
|
|
|
|
fn hpack_integer(out: &mut Vec<u8>, value: usize, prefix_bits: u8, first: u8) {
|
|
let limit = (1usize << prefix_bits) - 1;
|
|
if value < limit {
|
|
out.push(first | value as u8);
|
|
return;
|
|
}
|
|
out.push(first | limit as u8);
|
|
let mut rest = value - limit;
|
|
while rest >= 128 {
|
|
out.push((rest as u8 & 0x7f) | 0x80);
|
|
rest >>= 7;
|
|
}
|
|
out.push(rest as u8);
|
|
}
|
|
|
|
fn hpack_string(out: &mut Vec<u8>, value: &[u8]) {
|
|
// Huffman bit stays clear; the bootstrap encoder is deliberately simple.
|
|
hpack_integer(out, value.len(), 7, 0);
|
|
out.extend_from_slice(value);
|
|
}
|
|
|
|
fn hpack_literal(out: &mut Vec<u8>, name: &str, value: &str) {
|
|
// Literal Header Field without Indexing, new name (RFC 7541 section 6.2.2).
|
|
out.push(0);
|
|
hpack_string(out, name.as_bytes());
|
|
hpack_string(out, value.as_bytes());
|
|
}
|
|
|
|
fn h2_headers_frame<B>(request: &Request<B>) -> std::result::Result<Bytes, &'static str> {
|
|
if request.version() != http::Version::HTTP_11 {
|
|
return Err("h2c upgrade requires HTTP/1.1");
|
|
}
|
|
if !header_has_token(request.headers(), http::header::CONNECTION.as_str(), "upgrade")
|
|
|| !header_has_token(request.headers(), http::header::CONNECTION.as_str(), "http2-settings")
|
|
{
|
|
return Err("Connection must nominate Upgrade and HTTP2-Settings");
|
|
}
|
|
if !single_header(request.headers(), http::header::UPGRADE.as_str())?
|
|
.eq_ignore_ascii_case("h2c")
|
|
{
|
|
return Err("Upgrade must be exactly h2c");
|
|
}
|
|
let settings = single_header(request.headers(), "http2-settings")?;
|
|
let settings = URL_SAFE_NO_PAD
|
|
.decode(settings.as_bytes())
|
|
.map_err(|_| "invalid HTTP2-Settings base64url")?;
|
|
if settings.len() > 36 || settings.len() % 6 != 0 {
|
|
return Err("invalid HTTP2-Settings payload");
|
|
}
|
|
if request.headers().contains_key(http::header::TRANSFER_ENCODING) {
|
|
return Err("h2c upgrade request body is not supported");
|
|
}
|
|
if request.headers().get_all(http::header::CONTENT_LENGTH).iter().count() > 1 {
|
|
return Err("duplicate Content-Length on h2c upgrade");
|
|
}
|
|
if request
|
|
.headers()
|
|
.get(http::header::CONTENT_LENGTH)
|
|
.is_some_and(|value| value.as_bytes() != b"0")
|
|
{
|
|
return Err("h2c upgrade request body is not supported");
|
|
}
|
|
|
|
let authority = request
|
|
.uri()
|
|
.authority()
|
|
.map(|value| value.as_str())
|
|
.or_else(|| request.headers().get(http::header::HOST).and_then(|value| value.to_str().ok()))
|
|
.ok_or("h2c upgrade requires Host/:authority")?;
|
|
let path = request
|
|
.uri()
|
|
.path_and_query()
|
|
.map(|value| value.as_str())
|
|
.unwrap_or("/");
|
|
let method = request.method().as_str();
|
|
if method.len() > 32 || authority.len() > 1024 || path.len() > 8192 {
|
|
return Err("h2c bootstrap request fields exceed limits");
|
|
}
|
|
|
|
let mut block = Vec::with_capacity(1024);
|
|
hpack_literal(&mut block, ":method", method);
|
|
hpack_literal(&mut block, ":scheme", "http");
|
|
hpack_literal(&mut block, ":authority", authority);
|
|
hpack_literal(&mut block, ":path", path);
|
|
for (name, value) in request.headers() {
|
|
let name = name.as_str();
|
|
if matches!(
|
|
name,
|
|
"connection"
|
|
| "upgrade"
|
|
| "http2-settings"
|
|
| "host"
|
|
| "keep-alive"
|
|
| "proxy-connection"
|
|
| "transfer-encoding"
|
|
) || header_has_token(request.headers(), http::header::CONNECTION.as_str(), name)
|
|
{
|
|
continue;
|
|
}
|
|
if name == "te" && !value.as_bytes().eq_ignore_ascii_case(b"trailers") {
|
|
continue;
|
|
}
|
|
let value = value.to_str().map_err(|_| "non-text header in h2c bootstrap")?;
|
|
hpack_literal(&mut block, name, value);
|
|
}
|
|
if block.len() > 32_768 {
|
|
return Err("h2c bootstrap header block exceeds limit");
|
|
}
|
|
let length = block.len();
|
|
let mut frame = Vec::with_capacity(H2_FRAME_HEADER_LEN + length);
|
|
frame.extend_from_slice(&[
|
|
((length >> 16) & 0xff) as u8,
|
|
((length >> 8) & 0xff) as u8,
|
|
(length & 0xff) as u8,
|
|
0x1, // HEADERS
|
|
0x5, // END_STREAM | END_HEADERS
|
|
0,
|
|
0,
|
|
0,
|
|
1, // stream 1 is the HTTP/1.1 upgrade request
|
|
]);
|
|
frame.extend_from_slice(&block);
|
|
Ok(Bytes::from(frame))
|
|
}
|
|
|
|
/// After a successful 101 response the client sends the normal HTTP/2 client
|
|
/// connection preface. Hyper still needs to observe the upgrade request as
|
|
/// stream 1, so this adapter injects one synthetic HEADERS frame immediately
|
|
/// after the client's mandatory initial SETTINGS frame. All later streams are
|
|
/// handled by Hyper's normal HTTP/2 state machine.
|
|
struct H2cUpgradeIo<I> {
|
|
inner: I,
|
|
injected_headers: Bytes,
|
|
bootstrap: BytesMut,
|
|
ready: Bytes,
|
|
bootstrap_done: bool,
|
|
}
|
|
|
|
impl<I> H2cUpgradeIo<I> {
|
|
fn new(inner: I, injected_headers: Bytes) -> Self {
|
|
Self {
|
|
inner,
|
|
injected_headers,
|
|
bootstrap: BytesMut::with_capacity(4096),
|
|
ready: Bytes::new(),
|
|
bootstrap_done: false,
|
|
}
|
|
}
|
|
|
|
fn try_finish_bootstrap(&mut self) -> io::Result<()> {
|
|
if self.bootstrap_done || self.bootstrap.len() < HTTP2_PRIOR_KNOWLEDGE_PREFACE.len() + H2_FRAME_HEADER_LEN {
|
|
return Ok(());
|
|
}
|
|
if &self.bootstrap[..HTTP2_PRIOR_KNOWLEDGE_PREFACE.len()] != HTTP2_PRIOR_KNOWLEDGE_PREFACE {
|
|
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid HTTP/2 client preface after h2c upgrade"));
|
|
}
|
|
let offset = HTTP2_PRIOR_KNOWLEDGE_PREFACE.len();
|
|
let header = &self.bootstrap[offset..offset + H2_FRAME_HEADER_LEN];
|
|
let length = ((header[0] as usize) << 16) | ((header[1] as usize) << 8) | header[2] as usize;
|
|
if length > 16_384
|
|
|| header[3] != 0x4
|
|
|| header[4] != 0
|
|
|| header[5] != 0
|
|
|| header[6] != 0
|
|
|| header[7] != 0
|
|
|| header[8] != 0
|
|
{
|
|
return Err(io::Error::new(io::ErrorKind::InvalidData, "h2c client preface must begin with a non-ACK SETTINGS frame"));
|
|
}
|
|
let end = offset + H2_FRAME_HEADER_LEN + length;
|
|
if self.bootstrap.len() < end {
|
|
return Ok(());
|
|
}
|
|
let mut ready = BytesMut::with_capacity(self.bootstrap.len() + self.injected_headers.len());
|
|
ready.extend_from_slice(&self.bootstrap[..end]);
|
|
ready.extend_from_slice(&self.injected_headers);
|
|
ready.extend_from_slice(&self.bootstrap[end..]);
|
|
self.ready = ready.freeze();
|
|
self.bootstrap.clear();
|
|
self.bootstrap_done = true;
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
impl<I: AsyncRead + Unpin> AsyncRead for H2cUpgradeIo<I> {
|
|
fn poll_read(
|
|
mut self: Pin<&mut Self>,
|
|
cx: &mut TaskContext<'_>,
|
|
buf: &mut ReadBuf<'_>,
|
|
) -> Poll<io::Result<()>> {
|
|
loop {
|
|
if !self.ready.is_empty() {
|
|
let take = self.ready.len().min(buf.remaining());
|
|
buf.put_slice(&self.ready[..take]);
|
|
self.ready.advance(take);
|
|
return Poll::Ready(Ok(()));
|
|
}
|
|
if self.bootstrap_done {
|
|
return Pin::new(&mut self.inner).poll_read(cx, buf);
|
|
}
|
|
|
|
let mut temp = [0u8; 8192];
|
|
let mut temp_buf = ReadBuf::new(&mut temp);
|
|
match Pin::new(&mut self.inner).poll_read(cx, &mut temp_buf) {
|
|
Poll::Pending => return Poll::Pending,
|
|
Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
|
|
Poll::Ready(Ok(())) => {
|
|
let bytes = temp_buf.filled();
|
|
if bytes.is_empty() {
|
|
return Poll::Ready(Err(io::Error::new(
|
|
io::ErrorKind::UnexpectedEof,
|
|
"connection closed during h2c client preface",
|
|
)));
|
|
}
|
|
self.bootstrap.extend_from_slice(bytes);
|
|
if self.bootstrap.len() > 128 * 1024 {
|
|
return Poll::Ready(Err(io::Error::new(
|
|
io::ErrorKind::InvalidData,
|
|
"h2c bootstrap exceeds limit",
|
|
)));
|
|
}
|
|
if let Err(error) = self.try_finish_bootstrap() {
|
|
return Poll::Ready(Err(error));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<I: AsyncWrite + Unpin> AsyncWrite for H2cUpgradeIo<I> {
|
|
fn poll_write(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
|
|
Pin::new(&mut self.inner).poll_write(cx, buf)
|
|
}
|
|
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<io::Result<()>> {
|
|
Pin::new(&mut self.inner).poll_flush(cx)
|
|
}
|
|
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<io::Result<()>> {
|
|
Pin::new(&mut self.inner).poll_shutdown(cx)
|
|
}
|
|
}
|
|
|
|
/// Shared HTTP/2 activity state. Idle shutdown is allowed only when there are
|
|
/// no active streams and no recent frame traffic. Lifetime shutdown ignores idle
|
|
/// state but still uses the same GOAWAY -> drain -> TCP close sequence.
|
|
struct H2ConnectionState {
|
|
started: std::time::Instant,
|
|
last_activity_ms: AtomicU64,
|
|
active_streams: AtomicUsize,
|
|
notify: Notify,
|
|
}
|
|
impl H2ConnectionState {
|
|
fn new() -> Arc<Self> {
|
|
Arc::new(Self {
|
|
started: std::time::Instant::now(),
|
|
last_activity_ms: AtomicU64::new(0),
|
|
active_streams: AtomicUsize::new(0),
|
|
notify: Notify::new(),
|
|
})
|
|
}
|
|
fn elapsed_ms(&self) -> u64 {
|
|
self.started.elapsed().as_millis().min(u128::from(u64::MAX)) as u64
|
|
}
|
|
fn mark_activity(&self) {
|
|
self.last_activity_ms
|
|
.fetch_max(self.elapsed_ms(), Ordering::Relaxed);
|
|
self.notify.notify_waiters();
|
|
}
|
|
fn idle_for(&self) -> Duration {
|
|
Duration::from_millis(
|
|
self.elapsed_ms()
|
|
.saturating_sub(self.last_activity_ms.load(Ordering::Relaxed)),
|
|
)
|
|
}
|
|
fn is_idle(&self, idle: Duration) -> bool {
|
|
self.active_streams.load(Ordering::Acquire) == 0 && self.idle_for() >= idle
|
|
}
|
|
}
|
|
|
|
struct H2ActivityIo<I> {
|
|
inner: I,
|
|
state: Arc<H2ConnectionState>,
|
|
}
|
|
impl<I> H2ActivityIo<I> {
|
|
fn new(inner: I, state: Arc<H2ConnectionState>) -> Self {
|
|
Self { inner, state }
|
|
}
|
|
}
|
|
impl<I: Unpin> Unpin for H2ActivityIo<I> {}
|
|
impl<I: AsyncRead + Unpin> AsyncRead for H2ActivityIo<I> {
|
|
fn poll_read(
|
|
self: Pin<&mut Self>,
|
|
cx: &mut TaskContext<'_>,
|
|
buf: &mut ReadBuf<'_>,
|
|
) -> Poll<io::Result<()>> {
|
|
let this = self.get_mut();
|
|
let before = buf.filled().len();
|
|
match Pin::new(&mut this.inner).poll_read(cx, buf) {
|
|
Poll::Ready(Ok(())) => {
|
|
if buf.filled().len() > before {
|
|
this.state.mark_activity();
|
|
}
|
|
Poll::Ready(Ok(()))
|
|
}
|
|
other => other,
|
|
}
|
|
}
|
|
}
|
|
impl<I: AsyncWrite + Unpin> AsyncWrite for H2ActivityIo<I> {
|
|
fn poll_write(
|
|
self: Pin<&mut Self>,
|
|
cx: &mut TaskContext<'_>,
|
|
buf: &[u8],
|
|
) -> Poll<io::Result<usize>> {
|
|
let this = self.get_mut();
|
|
match Pin::new(&mut this.inner).poll_write(cx, buf) {
|
|
Poll::Ready(Ok(written)) => {
|
|
if written > 0 {
|
|
this.state.mark_activity();
|
|
}
|
|
Poll::Ready(Ok(written))
|
|
}
|
|
other => other,
|
|
}
|
|
}
|
|
fn poll_flush(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<io::Result<()>> {
|
|
Pin::new(&mut self.get_mut().inner).poll_flush(cx)
|
|
}
|
|
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<io::Result<()>> {
|
|
Pin::new(&mut self.get_mut().inner).poll_shutdown(cx)
|
|
}
|
|
}
|
|
|
|
struct H2StreamLease {
|
|
state: Arc<H2ConnectionState>,
|
|
}
|
|
impl H2StreamLease {
|
|
fn begin(state: Arc<H2ConnectionState>) -> Self {
|
|
state.active_streams.fetch_add(1, Ordering::AcqRel);
|
|
state.mark_activity();
|
|
Self { state }
|
|
}
|
|
}
|
|
impl Drop for H2StreamLease {
|
|
fn drop(&mut self) {
|
|
self.state.active_streams.fetch_sub(1, Ordering::AcqRel);
|
|
self.state.mark_activity();
|
|
}
|
|
}
|
|
|
|
struct H2TrackedBody<B> {
|
|
inner: Pin<Box<B>>,
|
|
stream: Option<H2StreamLease>,
|
|
}
|
|
impl<B> H2TrackedBody<B> {
|
|
fn new(inner: B, stream: H2StreamLease) -> Self {
|
|
Self {
|
|
inner: Box::pin(inner),
|
|
stream: Some(stream),
|
|
}
|
|
}
|
|
}
|
|
impl<B> HttpBody for H2TrackedBody<B>
|
|
where
|
|
B: HttpBody,
|
|
{
|
|
type Data = B::Data;
|
|
type Error = B::Error;
|
|
|
|
fn poll_frame(
|
|
self: Pin<&mut Self>,
|
|
cx: &mut TaskContext<'_>,
|
|
) -> Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
|
|
let this = self.get_mut();
|
|
match this.inner.as_mut().poll_frame(cx) {
|
|
Poll::Ready(None) => {
|
|
this.stream.take();
|
|
Poll::Ready(None)
|
|
}
|
|
Poll::Ready(Some(Err(error))) => {
|
|
this.stream.take();
|
|
Poll::Ready(Some(Err(error)))
|
|
}
|
|
other => other,
|
|
}
|
|
}
|
|
|
|
fn is_end_stream(&self) -> bool {
|
|
self.inner.as_ref().get_ref().is_end_stream()
|
|
}
|
|
|
|
fn size_hint(&self) -> http_body::SizeHint {
|
|
self.inner.as_ref().get_ref().size_hint()
|
|
}
|
|
}
|
|
|
|
async fn wait_for_h2_idle(state: Arc<H2ConnectionState>, idle: Duration) {
|
|
loop {
|
|
// Register for a notification before inspecting the state. This avoids
|
|
// losing the final stream-drop notification between the load and await.
|
|
let notified = state.notify.notified();
|
|
tokio::pin!(notified);
|
|
if state.active_streams.load(Ordering::Acquire) > 0 {
|
|
notified.as_mut().await;
|
|
continue;
|
|
}
|
|
let idle_for = state.idle_for();
|
|
if idle_for >= idle {
|
|
return;
|
|
}
|
|
tokio::select! {
|
|
_ = tokio::time::sleep(idle.saturating_sub(idle_for)) => {},
|
|
_ = notified.as_mut() => {},
|
|
}
|
|
}
|
|
}
|
|
|
|
struct ActiveHttp2Connection(Option<Arc<Metrics>>);
|
|
impl ActiveHttp2Connection {
|
|
fn begin(metrics: Option<Arc<Metrics>>) -> Self {
|
|
if let Some(metrics) = &metrics {
|
|
metrics
|
|
.http2_active_connections
|
|
.fetch_add(1, Ordering::Relaxed);
|
|
}
|
|
Self(metrics)
|
|
}
|
|
}
|
|
impl Drop for ActiveHttp2Connection {
|
|
fn drop(&mut self) {
|
|
if let Some(metrics) = &self.0 {
|
|
metrics
|
|
.http2_active_connections
|
|
.fetch_sub(1, Ordering::Relaxed);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn h2_prior_knowledge_prefix(bytes: &[u8]) -> bool {
|
|
bytes.len() <= HTTP2_PRIOR_KNOWLEDGE_PREFACE.len()
|
|
&& HTTP2_PRIOR_KNOWLEDGE_PREFACE.starts_with(bytes)
|
|
}
|
|
|
|
#[derive(Clone, Copy)]
|
|
pub struct PeerAddr(pub SocketAddr);
|
|
#[derive(Clone, Copy)]
|
|
pub struct LocalAddr(pub SocketAddr);
|
|
#[derive(Clone, Copy, Default)]
|
|
pub struct Transport {
|
|
pub tls: bool,
|
|
}
|
|
#[derive(Clone)]
|
|
pub struct ServerShutdown(pub CancellationToken);
|
|
#[derive(Clone)]
|
|
enum TlsMode {
|
|
Frontend(Arc<App>, bool),
|
|
Fixed(Arc<rustls::ServerConfig>, Option<Router>),
|
|
}
|
|
/// Upgraded tunnels hold this lease after Hyper finishes its HTTP connection.
|
|
/// This prevents WebSockets from escaping the public connection limit/counter.
|
|
pub struct ConnectionLease {
|
|
_permit: OwnedSemaphorePermit,
|
|
metrics: Option<Arc<Metrics>>,
|
|
}
|
|
impl Drop for ConnectionLease {
|
|
fn drop(&mut self) {
|
|
if let Some(m) = &self.metrics {
|
|
m.active_connections.fetch_sub(1, Ordering::Relaxed);
|
|
}
|
|
}
|
|
}
|
|
pub async fn serve(
|
|
listener: TcpListener,
|
|
router: Router,
|
|
shutdown: CancellationToken,
|
|
max_connections: usize,
|
|
grace: Duration,
|
|
metrics: Option<Arc<Metrics>>,
|
|
) -> Result<()> {
|
|
serve_inner(
|
|
listener,
|
|
router,
|
|
shutdown,
|
|
Arc::new(Semaphore::new(max_connections)),
|
|
grace,
|
|
metrics,
|
|
None,
|
|
false,
|
|
None,
|
|
Http2Connections::default(),
|
|
)
|
|
.await
|
|
}
|
|
pub async fn serve_public(
|
|
listener: TcpListener,
|
|
router: Router,
|
|
shutdown: CancellationToken,
|
|
permits: Arc<Semaphore>,
|
|
grace: Duration,
|
|
metrics: Arc<Metrics>,
|
|
tls: Option<Arc<App>>,
|
|
) -> Result<()> {
|
|
serve_frontend(
|
|
listener,
|
|
router,
|
|
shutdown,
|
|
permits,
|
|
grace,
|
|
metrics,
|
|
tls.clone().map(|app| (app, true)),
|
|
false,
|
|
tls,
|
|
)
|
|
.await
|
|
}
|
|
pub async fn serve_frontend(
|
|
listener: TcpListener,
|
|
router: Router,
|
|
shutdown: CancellationToken,
|
|
permits: Arc<Semaphore>,
|
|
grace: Duration,
|
|
metrics: Arc<Metrics>,
|
|
tls: Option<(Arc<App>, bool)>,
|
|
h2c: bool,
|
|
frontend_state: Option<Arc<App>>,
|
|
) -> Result<()> {
|
|
let http2_settings = frontend_state
|
|
.as_ref()
|
|
.map(|state| state.config.load().envelope.config.http2)
|
|
.unwrap_or_default();
|
|
serve_frontend_with_http2(
|
|
listener,
|
|
router,
|
|
shutdown,
|
|
permits,
|
|
grace,
|
|
metrics,
|
|
tls,
|
|
h2c,
|
|
frontend_state,
|
|
http2_settings,
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// Start one compiled frontend listener with its immutable per-listener H2
|
|
/// lifecycle policy. The public `serve_frontend` wrapper retains the legacy
|
|
/// global fallback for callers that do not provide a compiled listener plan.
|
|
pub async fn serve_frontend_with_http2(
|
|
listener: TcpListener,
|
|
router: Router,
|
|
shutdown: CancellationToken,
|
|
permits: Arc<Semaphore>,
|
|
grace: Duration,
|
|
metrics: Arc<Metrics>,
|
|
tls: Option<(Arc<App>, bool)>,
|
|
h2c: bool,
|
|
frontend_state: Option<Arc<App>>,
|
|
http2_settings: Http2Connections,
|
|
) -> Result<()> {
|
|
serve_inner(
|
|
listener,
|
|
router,
|
|
shutdown,
|
|
permits,
|
|
grace,
|
|
Some(metrics),
|
|
tls.map(|(app, http2)| TlsMode::Frontend(app, http2)),
|
|
h2c,
|
|
frontend_state,
|
|
http2_settings,
|
|
)
|
|
.await
|
|
}
|
|
pub async fn serve_tls(
|
|
listener: TcpListener,
|
|
router: Router,
|
|
shutdown: CancellationToken,
|
|
max_connections: usize,
|
|
grace: Duration,
|
|
tls: Arc<rustls::ServerConfig>,
|
|
) -> Result<()> {
|
|
serve_inner(
|
|
listener,
|
|
router,
|
|
shutdown,
|
|
Arc::new(Semaphore::new(max_connections)),
|
|
grace,
|
|
None,
|
|
Some(TlsMode::Fixed(tls, None)),
|
|
false,
|
|
None,
|
|
Http2Connections::default(),
|
|
)
|
|
.await
|
|
}
|
|
/// Serve an HTTPS socket while turning accidental/plain HTTP requests on the
|
|
/// same port into redirects. No administration handler is exposed through the
|
|
/// plaintext router.
|
|
pub async fn serve_tls_with_plain_redirect(
|
|
listener: TcpListener,
|
|
router: Router,
|
|
plaintext_router: Router,
|
|
shutdown: CancellationToken,
|
|
max_connections: usize,
|
|
grace: Duration,
|
|
tls: Arc<rustls::ServerConfig>,
|
|
) -> Result<()> {
|
|
serve_inner(
|
|
listener,
|
|
router,
|
|
shutdown,
|
|
Arc::new(Semaphore::new(max_connections)),
|
|
grace,
|
|
None,
|
|
Some(TlsMode::Fixed(tls, Some(plaintext_router))),
|
|
false,
|
|
None,
|
|
Http2Connections::default(),
|
|
)
|
|
.await
|
|
}
|
|
async fn serve_inner(
|
|
listener: TcpListener,
|
|
router: Router,
|
|
shutdown: CancellationToken,
|
|
permits: Arc<Semaphore>,
|
|
grace: Duration,
|
|
metrics: Option<Arc<Metrics>>,
|
|
tls: Option<TlsMode>,
|
|
h2c: bool,
|
|
frontend_state: Option<Arc<App>>,
|
|
http2_settings: Http2Connections,
|
|
) -> Result<()> {
|
|
let mut tasks = JoinSet::new();
|
|
loop {
|
|
tokio::select! {
|
|
biased;
|
|
_ = shutdown.cancelled() => break,
|
|
_ = tasks.join_next(), if !tasks.is_empty() => {},
|
|
accepted = listener.accept() => {
|
|
let (stream, peer) = accepted.context("listener accept failed")?;
|
|
// Never substitute the wildcard listener address or a forwarded
|
|
// header for the destination IP of this accepted connection.
|
|
let local = stream.local_addr().context("accepted local address unavailable")?;
|
|
let Ok(permit) = permits.clone().try_acquire_owned() else {
|
|
if let Some(m) = &metrics { m.rejected_connections.fetch_add(1, Ordering::Relaxed); }
|
|
drop(stream); continue;
|
|
};
|
|
let _ = stream.set_nodelay(true);
|
|
if let Some(m) = &metrics {
|
|
m.connections.fetch_add(1, Ordering::Relaxed); m.active_connections.fetch_add(1, Ordering::Relaxed);
|
|
}
|
|
let lease = Arc::new(ConnectionLease { _permit: permit, metrics: metrics.clone() });
|
|
let router = router.clone(); let stop = shutdown.clone(); let tls = tls.clone(); let metrics = metrics.clone();
|
|
let frontend_state = frontend_state.clone();
|
|
tasks.spawn(async move {
|
|
match tls {
|
|
Some(TlsMode::Frontend(state, http2)) => {
|
|
// Read ClientHello before committing to ALPN so a vhost can opt out
|
|
// of HTTP/2 or narrow TLS policy while sharing one TLS listener.
|
|
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
|
let acceptor = tokio_rustls::LazyConfigAcceptor::new(
|
|
rustls::server::Acceptor::default(),
|
|
stream,
|
|
);
|
|
let started = tokio::select! {
|
|
_ = stop.cancelled() => return,
|
|
started = tokio::time::timeout_at(deadline, acceptor) => started,
|
|
};
|
|
let Ok(Ok(start)) = started else {
|
|
state.metrics.tls_handshake_errors.fetch_add(1, Ordering::Relaxed);
|
|
return;
|
|
};
|
|
let config = {
|
|
let snapshot = state.config.load();
|
|
let host = start
|
|
.client_hello()
|
|
.server_name()
|
|
.and_then(|name| crate::config::request_host(name).ok());
|
|
let frontend = host
|
|
.as_deref()
|
|
.and_then(|host| snapshot.frontend(host, local, true));
|
|
let local_host = host
|
|
.as_deref()
|
|
.and_then(|host| snapshot.not_found_host_on(host, local, true));
|
|
if let Some(frontend) = frontend {
|
|
if http2 && frontend.http2_enabled {
|
|
frontend.tls.clone()
|
|
} else {
|
|
frontend.tls_http1.clone()
|
|
}
|
|
} else if http2 && local_host.is_none_or(|item| item.http2_enabled) {
|
|
snapshot.tls.clone()
|
|
} else {
|
|
snapshot.tls_http1.clone()
|
|
}
|
|
};
|
|
let Some(config) = config else { return; };
|
|
let accepted = tokio::select! {
|
|
_ = stop.cancelled() => return,
|
|
accepted = tokio::time::timeout_at(deadline, start.into_stream(config)) => accepted,
|
|
};
|
|
match accepted {
|
|
Ok(Ok(stream)) => {
|
|
let h2 = stream.get_ref().1.alpn_protocol() == Some(b"h2".as_slice());
|
|
state.metrics.tls_connections.fetch_add(1, Ordering::Relaxed);
|
|
if h2 { state.metrics.http2_connections.fetch_add(1, Ordering::Relaxed); }
|
|
let extended_connect = h2 && state.config.load().websocket_rfc8441_on(local, true);
|
|
connection(
|
|
stream,
|
|
router,
|
|
peer,
|
|
local,
|
|
Transport { tls: true },
|
|
h2,
|
|
extended_connect,
|
|
false,
|
|
None,
|
|
lease,
|
|
stop,
|
|
grace,
|
|
http2_settings,
|
|
)
|
|
.await;
|
|
},
|
|
_ => { state.metrics.tls_handshake_errors.fetch_add(1, Ordering::Relaxed); },
|
|
}
|
|
}
|
|
Some(TlsMode::Fixed(config, plaintext_router)) => {
|
|
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
|
if let Some(plain) = plaintext_router {
|
|
let mut first = [0u8; 1];
|
|
let peeked = tokio::select! {
|
|
_ = stop.cancelled() => return,
|
|
peeked = tokio::time::timeout_at(deadline, stream.peek(&mut first)) => peeked,
|
|
};
|
|
match peeked {
|
|
Ok(Ok(n)) if n > 0 && first[0] != 0x16 => {
|
|
connection(
|
|
stream,
|
|
plain,
|
|
peer,
|
|
local,
|
|
Transport { tls: false },
|
|
false,
|
|
false,
|
|
false,
|
|
None,
|
|
lease,
|
|
stop,
|
|
grace,
|
|
Http2Connections::default(),
|
|
)
|
|
.await;
|
|
return;
|
|
}
|
|
Ok(Ok(n)) if n > 0 => {}
|
|
_ => return,
|
|
}
|
|
}
|
|
let acceptor = tokio_rustls::TlsAcceptor::from(config);
|
|
let accepted = tokio::select! {
|
|
_ = stop.cancelled() => return,
|
|
accepted = tokio::time::timeout_at(deadline, acceptor.accept(stream)) => accepted,
|
|
};
|
|
if let Ok(Ok(stream)) = accepted {
|
|
let h2 = stream.get_ref().1.alpn_protocol() == Some(b"h2".as_slice());
|
|
connection(
|
|
stream,
|
|
router,
|
|
peer,
|
|
local,
|
|
Transport { tls: true },
|
|
h2,
|
|
false,
|
|
false,
|
|
None,
|
|
lease,
|
|
stop,
|
|
grace,
|
|
Http2Connections::default(),
|
|
)
|
|
.await;
|
|
}
|
|
}
|
|
None => {
|
|
let prior_h2 = if h2c {
|
|
let detected = tokio::time::timeout(Duration::from_secs(10), async {
|
|
let mut preface = [0u8; HTTP2_PRIOR_KNOWLEDGE_PREFACE.len()];
|
|
loop {
|
|
let n = stream.peek(&mut preface).await?;
|
|
if n == 0 {
|
|
return Ok::<bool, std::io::Error>(false);
|
|
}
|
|
let prefix = &preface[..n.min(HTTP2_PRIOR_KNOWLEDGE_PREFACE.len())];
|
|
if !h2_prior_knowledge_prefix(prefix) {
|
|
return Ok(false);
|
|
}
|
|
if n >= HTTP2_PRIOR_KNOWLEDGE_PREFACE.len() {
|
|
return Ok(prefix == HTTP2_PRIOR_KNOWLEDGE_PREFACE);
|
|
}
|
|
tokio::task::yield_now().await;
|
|
}
|
|
}).await;
|
|
matches!(detected, Ok(Ok(true)))
|
|
} else {
|
|
false
|
|
};
|
|
if prior_h2 {
|
|
if let Some(m) = &metrics { m.http2_connections.fetch_add(1, Ordering::Relaxed); }
|
|
}
|
|
let extended_connect = prior_h2
|
|
&& frontend_state
|
|
.as_ref()
|
|
.is_some_and(|state| state.config.load().websocket_rfc8441_on(local, false));
|
|
connection(
|
|
stream,
|
|
router,
|
|
peer,
|
|
local,
|
|
Transport { tls: false },
|
|
prior_h2,
|
|
extended_connect,
|
|
h2c && !prior_h2,
|
|
frontend_state,
|
|
lease,
|
|
stop,
|
|
grace,
|
|
http2_settings,
|
|
)
|
|
.await;
|
|
}
|
|
}
|
|
});
|
|
},
|
|
}
|
|
}
|
|
let _ = tokio::time::timeout(grace + Duration::from_secs(1), async {
|
|
while tasks.join_next().await.is_some() {}
|
|
})
|
|
.await;
|
|
tasks.abort_all();
|
|
while tasks.join_next().await.is_some() {}
|
|
Ok(())
|
|
}
|
|
#[allow(clippy::too_many_arguments)]
|
|
async fn connection<I>(
|
|
stream: I,
|
|
router: Router,
|
|
peer: SocketAddr,
|
|
local: SocketAddr,
|
|
transport: Transport,
|
|
h2: bool,
|
|
extended_connect: bool,
|
|
h2c_upgrade: bool,
|
|
h2c_state: Option<Arc<App>>,
|
|
lease: Arc<ConnectionLease>,
|
|
stop: CancellationToken,
|
|
grace: Duration,
|
|
http2_settings: Http2Connections,
|
|
) where
|
|
I: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
|
{
|
|
if h2 {
|
|
http2_connection(
|
|
stream,
|
|
router,
|
|
peer,
|
|
local,
|
|
transport,
|
|
extended_connect,
|
|
lease,
|
|
stop,
|
|
grace,
|
|
http2_settings,
|
|
)
|
|
.await;
|
|
return;
|
|
}
|
|
let request_stop = stop.clone();
|
|
let service = service_fn(move |mut request: Request<Incoming>| {
|
|
let router = router.clone();
|
|
let request_stop = request_stop.clone();
|
|
let request_lease = lease.clone();
|
|
let upgrade_state = h2c_state.clone();
|
|
async move {
|
|
request.extensions_mut().insert(PeerAddr(peer));
|
|
request.extensions_mut().insert(LocalAddr(local));
|
|
request.extensions_mut().insert(transport);
|
|
request.extensions_mut().insert(request_lease.clone());
|
|
request
|
|
.extensions_mut()
|
|
.insert(ServerShutdown(request_stop.clone()));
|
|
|
|
if h2c_attempt(&request) {
|
|
if !h2c_upgrade {
|
|
return Ok::<_, Infallible>(
|
|
Response::builder()
|
|
.status(http::StatusCode::BAD_REQUEST)
|
|
.header(http::header::CACHE_CONTROL, "no-store")
|
|
.body(Body::from("h2c upgrade is disabled on this listener"))
|
|
.expect("static h2c disabled response"),
|
|
);
|
|
}
|
|
let injected_headers = match h2_headers_frame(&request) {
|
|
Ok(frame) => frame,
|
|
Err(message) => {
|
|
return Ok::<_, Infallible>(
|
|
Response::builder()
|
|
.status(http::StatusCode::BAD_REQUEST)
|
|
.header(http::header::CACHE_CONTROL, "no-store")
|
|
.body(Body::from(message))
|
|
.expect("static h2c validation response"),
|
|
)
|
|
}
|
|
};
|
|
let on_upgrade = hyper::upgrade::on(&mut request);
|
|
let upgrade_router = router.clone();
|
|
let upgrade_stop = request_stop.clone();
|
|
let upgrade_lease = request_lease.clone();
|
|
let upgrade_state_for_task = upgrade_state.clone();
|
|
let task = async move {
|
|
match on_upgrade.await {
|
|
Ok(upgraded) => {
|
|
if let Some(state) = &upgrade_state_for_task {
|
|
state.metrics.http2_connections.fetch_add(1, Ordering::Relaxed);
|
|
}
|
|
let extended_connect = upgrade_state_for_task.as_ref().is_some_and(|state| {
|
|
state.config.load().websocket_rfc8441_on(local, false)
|
|
});
|
|
let io = H2cUpgradeIo::new(TokioIo::new(upgraded), injected_headers);
|
|
http2_connection(
|
|
io,
|
|
upgrade_router,
|
|
peer,
|
|
local,
|
|
Transport { tls: false },
|
|
extended_connect,
|
|
upgrade_lease,
|
|
upgrade_stop,
|
|
grace,
|
|
http2_settings,
|
|
)
|
|
.await;
|
|
}
|
|
Err(error) => tracing::debug!(%error, "h2c upgrade transport failed"),
|
|
}
|
|
};
|
|
if let Some(state) = upgrade_state {
|
|
state.tunnels.spawn(task);
|
|
} else {
|
|
tokio::spawn(task);
|
|
}
|
|
return Ok::<_, Infallible>(
|
|
Response::builder()
|
|
.status(http::StatusCode::SWITCHING_PROTOCOLS)
|
|
.header(http::header::CONNECTION, "Upgrade")
|
|
.header(http::header::UPGRADE, "h2c")
|
|
.body(Body::empty())
|
|
.expect("static h2c switching response"),
|
|
);
|
|
}
|
|
|
|
router.oneshot(request.map(Body::new)).await
|
|
}
|
|
});
|
|
let mut builder = http1::Builder::new();
|
|
builder
|
|
.timer(TokioTimer::new())
|
|
.keep_alive(true)
|
|
.max_headers(64)
|
|
.max_buf_size(65_536)
|
|
.header_read_timeout(Duration::from_secs(10));
|
|
let conn = builder
|
|
.serve_connection(TokioIo::new(stream), service)
|
|
.with_upgrades();
|
|
tokio::pin!(conn);
|
|
tokio::select! {
|
|
result = &mut conn => { if let Err(error) = result { tracing::debug!(%error, "HTTP/1 connection closed"); } },
|
|
_ = stop.cancelled() => {
|
|
conn.as_mut().graceful_shutdown(); let _ = tokio::time::timeout(grace, &mut conn).await;
|
|
},
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug)]
|
|
enum H2ShutdownCause {
|
|
Idle,
|
|
Lifetime,
|
|
Process,
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
async fn http2_connection<I>(
|
|
stream: I,
|
|
router: Router,
|
|
peer: SocketAddr,
|
|
local: SocketAddr,
|
|
transport: Transport,
|
|
extended_connect: bool,
|
|
lease: Arc<ConnectionLease>,
|
|
stop: CancellationToken,
|
|
grace: Duration,
|
|
settings: Http2Connections,
|
|
) where
|
|
I: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
|
{
|
|
let request_stop = stop.clone();
|
|
let connection_metrics = lease.metrics.clone();
|
|
let activity = H2ConnectionState::new();
|
|
let service_activity = activity.clone();
|
|
let service = service_fn(move |mut request: Request<Incoming>| {
|
|
let stream_lease = H2StreamLease::begin(service_activity.clone());
|
|
request.extensions_mut().insert(PeerAddr(peer));
|
|
request.extensions_mut().insert(LocalAddr(local));
|
|
request.extensions_mut().insert(transport);
|
|
request.extensions_mut().insert(lease.clone());
|
|
request
|
|
.extensions_mut()
|
|
.insert(ServerShutdown(request_stop.clone()));
|
|
let router = router.clone();
|
|
async move {
|
|
router
|
|
.oneshot(request.map(Body::new))
|
|
.await
|
|
.map(|response| {
|
|
response.map(|body| Body::new(H2TrackedBody::new(body, stream_lease)))
|
|
})
|
|
}
|
|
});
|
|
let _active_http2 = ActiveHttp2Connection::begin(connection_metrics);
|
|
let stream = H2ActivityIo::new(stream, activity.clone());
|
|
let mut builder = http2::Builder::new(TokioExecutor::new());
|
|
builder
|
|
.timer(TokioTimer::new())
|
|
.max_concurrent_streams(64)
|
|
.max_header_list_size(32_768)
|
|
.max_send_buf_size(65_536)
|
|
.initial_stream_window_size(65_535)
|
|
.initial_connection_window_size(1_048_576);
|
|
if extended_connect {
|
|
builder.enable_connect_protocol();
|
|
}
|
|
let conn = builder.serve_connection(TokioIo::new(stream), service);
|
|
tokio::pin!(conn);
|
|
|
|
let max_lifetime = settings.max_connection_lifetime();
|
|
let lifetime = async move {
|
|
if let Some(duration) = max_lifetime {
|
|
tokio::time::sleep(duration).await;
|
|
} else {
|
|
std::future::pending::<()>().await;
|
|
}
|
|
};
|
|
tokio::pin!(lifetime);
|
|
|
|
let cause = loop {
|
|
tokio::select! {
|
|
result = &mut conn => {
|
|
if let Err(error) = result {
|
|
tracing::debug!(%error, "HTTP/2 connection closed");
|
|
}
|
|
return;
|
|
},
|
|
_ = stop.cancelled() => break H2ShutdownCause::Process,
|
|
_ = &mut lifetime => break H2ShutdownCause::Lifetime,
|
|
_ = wait_for_h2_idle(activity.clone(), settings.idle_timeout()) => {
|
|
// Re-check immediately before GOAWAY so activity racing the timer
|
|
// cannot turn a busy connection into an idle shutdown.
|
|
if activity.is_idle(settings.idle_timeout()) {
|
|
break H2ShutdownCause::Idle;
|
|
}
|
|
},
|
|
}
|
|
};
|
|
|
|
// Hyper's graceful shutdown sends GOAWAY and refuses newer streams while
|
|
// continuing to poll streams that were already accepted.
|
|
conn.as_mut().graceful_shutdown();
|
|
let configured_drain = settings.drain_timeout();
|
|
let drain = if matches!(cause, H2ShutdownCause::Process) {
|
|
std::cmp::min(configured_drain, grace)
|
|
} else {
|
|
configured_drain
|
|
};
|
|
match tokio::time::timeout(drain, &mut conn).await {
|
|
Ok(Ok(())) => tracing::debug!(?cause, "HTTP/2 graceful shutdown drained"),
|
|
Ok(Err(error)) => tracing::debug!(?cause, %error, "HTTP/2 graceful shutdown ended with protocol error"),
|
|
Err(_) => tracing::debug!(?cause, drain_seconds = drain.as_secs_f64(), "HTTP/2 drain timeout reached; closing transport"),
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod h2c_tests {
|
|
use super::*;
|
|
use axum::routing::get;
|
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
|
|
#[test]
|
|
fn h2_prior_knowledge_preface_is_distinguished_from_http1() {
|
|
assert!(h2_prior_knowledge_prefix(b"P"));
|
|
assert!(h2_prior_knowledge_prefix(b"PRI * HTTP/2.0\r\n"));
|
|
assert!(h2_prior_knowledge_prefix(HTTP2_PRIOR_KNOWLEDGE_PREFACE));
|
|
assert!(!h2_prior_knowledge_prefix(b"POST / HTTP/1.1\r\n"));
|
|
assert!(!h2_prior_knowledge_prefix(b"GET / HTTP/1.1\r\n"));
|
|
}
|
|
|
|
#[test]
|
|
fn h2c_upgrade_validation_rejects_missing_settings_and_bodies() {
|
|
let missing = Request::builder()
|
|
.method("GET")
|
|
.uri("/upgrade")
|
|
.version(http::Version::HTTP_11)
|
|
.header("host", "localhost")
|
|
.header("connection", "Upgrade")
|
|
.header("upgrade", "h2c")
|
|
.body(())
|
|
.unwrap();
|
|
assert!(h2_headers_frame(&missing).is_err());
|
|
|
|
let body = Request::builder()
|
|
.method("POST")
|
|
.uri("/upgrade")
|
|
.version(http::Version::HTTP_11)
|
|
.header("host", "localhost")
|
|
.header("connection", "Upgrade, HTTP2-Settings")
|
|
.header("upgrade", "h2c")
|
|
.header("http2-settings", "")
|
|
.header("content-length", "1")
|
|
.body(())
|
|
.unwrap();
|
|
assert!(h2_headers_frame(&body).is_err());
|
|
}
|
|
|
|
async fn read_http1_head(stream: &mut tokio::net::TcpStream) -> Vec<u8> {
|
|
let mut all = Vec::new();
|
|
let mut byte = [0u8; 1];
|
|
while !all.ends_with(b"\r\n\r\n") {
|
|
stream.read_exact(&mut byte).await.unwrap();
|
|
all.push(byte[0]);
|
|
assert!(all.len() < 8192, "HTTP/1 response head exceeded test bound");
|
|
}
|
|
all
|
|
}
|
|
|
|
async fn try_read_h2_frame(
|
|
stream: &mut tokio::net::TcpStream,
|
|
) -> io::Result<(u8, u8, u32, Vec<u8>)> {
|
|
let mut header = [0u8; 9];
|
|
stream.read_exact(&mut header).await?;
|
|
let length = ((header[0] as usize) << 16) | ((header[1] as usize) << 8) | header[2] as usize;
|
|
if length > 64 * 1024 {
|
|
return Err(io::Error::new(io::ErrorKind::InvalidData, "unexpectedly large HTTP/2 test frame"));
|
|
}
|
|
let mut payload = vec![0u8; length];
|
|
stream.read_exact(&mut payload).await?;
|
|
let stream_id = u32::from_be_bytes([header[5], header[6], header[7], header[8]]) & 0x7fff_ffff;
|
|
Ok((header[3], header[4], stream_id, payload))
|
|
}
|
|
|
|
async fn read_h2_frame(stream: &mut tokio::net::TcpStream) -> (u8, u8, u32, Vec<u8>) {
|
|
try_read_h2_frame(stream).await.unwrap()
|
|
}
|
|
|
|
async fn start_direct_h2(
|
|
router: Router,
|
|
settings: Http2Connections,
|
|
) -> (tokio::net::TcpStream, CancellationToken, tokio::task::JoinHandle<()>) {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let stop = CancellationToken::new();
|
|
let server_stop = stop.clone();
|
|
let server = tokio::spawn(async move {
|
|
let (stream, peer) = listener.accept().await.unwrap();
|
|
let local = stream.local_addr().unwrap();
|
|
let permit = Arc::new(Semaphore::new(1)).acquire_owned().await.unwrap();
|
|
let lease = Arc::new(ConnectionLease {
|
|
_permit: permit,
|
|
metrics: None,
|
|
});
|
|
http2_connection(
|
|
stream,
|
|
router,
|
|
peer,
|
|
local,
|
|
Transport { tls: false },
|
|
false,
|
|
lease,
|
|
server_stop,
|
|
Duration::from_secs(5),
|
|
settings,
|
|
)
|
|
.await;
|
|
});
|
|
let mut client = tokio::net::TcpStream::connect(address).await.unwrap();
|
|
client.write_all(HTTP2_PRIOR_KNOWLEDGE_PREFACE).await.unwrap();
|
|
client
|
|
.write_all(&[0, 0, 0, 0x4, 0, 0, 0, 0, 0])
|
|
.await
|
|
.unwrap();
|
|
(client, stop, server)
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn http1_h2c_upgrade_bootstraps_request_as_http2_stream_one() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let stop = CancellationToken::new();
|
|
let server_stop = stop.clone();
|
|
let router = Router::new().route("/upgrade", get(|| async { "h2c-ok" }));
|
|
let server = tokio::spawn(async move {
|
|
serve_inner(
|
|
listener,
|
|
router,
|
|
server_stop,
|
|
Arc::new(Semaphore::new(8)),
|
|
Duration::from_secs(1),
|
|
None,
|
|
None,
|
|
true,
|
|
None,
|
|
)
|
|
.await
|
|
.unwrap();
|
|
});
|
|
|
|
let mut client = tokio::net::TcpStream::connect(address).await.unwrap();
|
|
client
|
|
.write_all(
|
|
b"GET /upgrade HTTP/1.1\r\nHost: localhost\r\nConnection: Upgrade, HTTP2-Settings\r\nUpgrade: h2c\r\nHTTP2-Settings: \r\n\r\n",
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let response = read_http1_head(&mut client).await;
|
|
let response = String::from_utf8(response).unwrap();
|
|
assert!(response.starts_with("HTTP/1.1 101"), "{response}");
|
|
assert!(response.to_ascii_lowercase().contains("upgrade: h2c"));
|
|
|
|
client.write_all(HTTP2_PRIOR_KNOWLEDGE_PREFACE).await.unwrap();
|
|
// Client connection preface must begin with a non-ACK SETTINGS frame.
|
|
client
|
|
.write_all(&[0, 0, 0, 0x4, 0, 0, 0, 0, 0])
|
|
.await
|
|
.unwrap();
|
|
|
|
let mut saw_stream_one_headers = false;
|
|
let mut body = Vec::new();
|
|
for _ in 0..16 {
|
|
let (kind, flags, stream_id, payload) = tokio::time::timeout(
|
|
Duration::from_secs(2),
|
|
read_h2_frame(&mut client),
|
|
)
|
|
.await
|
|
.expect("HTTP/2 response frame timeout");
|
|
if kind == 0x1 && stream_id == 1 {
|
|
saw_stream_one_headers = true;
|
|
}
|
|
if kind == 0x0 && stream_id == 1 {
|
|
body.extend_from_slice(&payload);
|
|
if flags & 0x1 != 0 {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
assert!(saw_stream_one_headers, "missing stream 1 response HEADERS");
|
|
assert_eq!(body, b"h2c-ok");
|
|
|
|
stop.cancel();
|
|
server.await.unwrap();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn h2_idle_connection_sends_goaway_before_closing() {
|
|
let settings = Http2Connections {
|
|
idle_timeout_seconds: 1,
|
|
max_connection_lifetime_seconds: 0,
|
|
drain_timeout_seconds: 1,
|
|
};
|
|
let (mut client, stop, server) = start_direct_h2(Router::new(), settings).await;
|
|
let saw_goaway = tokio::time::timeout(Duration::from_secs(3), async {
|
|
loop {
|
|
match try_read_h2_frame(&mut client).await {
|
|
Ok((0x7, _, 0, _)) => return true,
|
|
Ok(_) => {}
|
|
Err(_) => return false,
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("idle HTTP/2 shutdown did not complete");
|
|
assert!(saw_goaway, "idle shutdown must send GOAWAY");
|
|
stop.cancel();
|
|
server.await.unwrap();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn h2_idle_wait_does_not_close_an_active_stream() {
|
|
let state = H2ConnectionState::new();
|
|
let stream = H2StreamLease::begin(state.clone());
|
|
tokio::time::sleep(Duration::from_millis(30)).await;
|
|
assert!(tokio::time::timeout(
|
|
Duration::from_millis(30),
|
|
wait_for_h2_idle(state.clone(), Duration::from_millis(10))
|
|
)
|
|
.await
|
|
.is_err());
|
|
drop(stream);
|
|
tokio::time::timeout(
|
|
Duration::from_millis(100),
|
|
wait_for_h2_idle(state, Duration::from_millis(10)),
|
|
)
|
|
.await
|
|
.expect("idle shutdown should become eligible after the stream finishes");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn h2_lifetime_goaway_is_bounded_by_drain_timeout() {
|
|
let blocker = Arc::new(Notify::new());
|
|
let wait = blocker.clone();
|
|
let router = Router::new().route(
|
|
"/hold",
|
|
get(move || {
|
|
let wait = wait.clone();
|
|
async move {
|
|
wait.notified().await;
|
|
"done"
|
|
}
|
|
}),
|
|
);
|
|
let settings = Http2Connections {
|
|
idle_timeout_seconds: 60,
|
|
max_connection_lifetime_seconds: 1,
|
|
drain_timeout_seconds: 1,
|
|
};
|
|
let (mut client, _stop, server) = start_direct_h2(router, settings).await;
|
|
let upgrade = Request::builder()
|
|
.method("GET")
|
|
.uri("/hold")
|
|
.version(http::Version::HTTP_11)
|
|
.header("host", "localhost")
|
|
.header("connection", "Upgrade, HTTP2-Settings")
|
|
.header("upgrade", "h2c")
|
|
.header("http2-settings", "")
|
|
.body(())
|
|
.unwrap();
|
|
client.write_all(&h2_headers_frame(&upgrade).unwrap()).await.unwrap();
|
|
|
|
let saw_goaway_and_close = tokio::time::timeout(Duration::from_secs(4), async {
|
|
let mut saw_goaway = false;
|
|
loop {
|
|
match try_read_h2_frame(&mut client).await {
|
|
Ok((0x7, _, 0, _)) => saw_goaway = true,
|
|
Ok(_) => {}
|
|
Err(error) if matches!(error.kind(), io::ErrorKind::UnexpectedEof | io::ErrorKind::ConnectionReset) => {
|
|
return saw_goaway;
|
|
}
|
|
Err(_) => return saw_goaway,
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("lifetime/drain shutdown exceeded its bound");
|
|
assert!(saw_goaway_and_close, "lifetime shutdown must send GOAWAY before drain timeout closes TCP");
|
|
server.await.unwrap();
|
|
drop(blocker);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn h2_lifetime_allows_active_stream_to_finish_during_drain() {
|
|
let blocker = Arc::new(Notify::new());
|
|
let wait = blocker.clone();
|
|
let router = Router::new().route(
|
|
"/hold",
|
|
get(move || {
|
|
let wait = wait.clone();
|
|
async move {
|
|
wait.notified().await;
|
|
"done"
|
|
}
|
|
}),
|
|
);
|
|
let settings = Http2Connections {
|
|
idle_timeout_seconds: 60,
|
|
max_connection_lifetime_seconds: 1,
|
|
drain_timeout_seconds: 3,
|
|
};
|
|
let (mut client, _stop, server) = start_direct_h2(router, settings).await;
|
|
let request = Request::builder()
|
|
.method("GET")
|
|
.uri("/hold")
|
|
.version(http::Version::HTTP_11)
|
|
.header("host", "localhost")
|
|
.header("connection", "Upgrade, HTTP2-Settings")
|
|
.header("upgrade", "h2c")
|
|
.header("http2-settings", "")
|
|
.body(())
|
|
.unwrap();
|
|
client.write_all(&h2_headers_frame(&request).unwrap()).await.unwrap();
|
|
let release = blocker.clone();
|
|
tokio::spawn(async move {
|
|
tokio::time::sleep(Duration::from_millis(1_250)).await;
|
|
release.notify_waiters();
|
|
});
|
|
|
|
let (saw_goaway, saw_end_stream) = tokio::time::timeout(Duration::from_secs(4), async {
|
|
let mut saw_goaway = false;
|
|
let mut saw_end_stream = false;
|
|
loop {
|
|
match try_read_h2_frame(&mut client).await {
|
|
Ok((0x7, _, 0, _)) => saw_goaway = true,
|
|
Ok((kind, flags, 1, _)) if (kind == 0x0 || kind == 0x1) && flags & 0x1 != 0 => {
|
|
saw_end_stream = true;
|
|
if saw_goaway { return (true, true); }
|
|
}
|
|
Ok(_) => {}
|
|
Err(_) => return (saw_goaway, saw_end_stream),
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("active stream did not drain within the configured bound");
|
|
assert!(saw_goaway, "lifetime shutdown must send GOAWAY");
|
|
assert!(saw_end_stream, "an accepted stream must be allowed to finish during drain");
|
|
server.await.unwrap();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn malformed_h2c_upgrade_is_rejected_before_switching_protocols() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let stop = CancellationToken::new();
|
|
let server_stop = stop.clone();
|
|
let server = tokio::spawn(async move {
|
|
serve_inner(
|
|
listener,
|
|
Router::new(),
|
|
server_stop,
|
|
Arc::new(Semaphore::new(8)),
|
|
Duration::from_secs(1),
|
|
None,
|
|
None,
|
|
true,
|
|
None,
|
|
)
|
|
.await
|
|
.unwrap();
|
|
});
|
|
let mut client = tokio::net::TcpStream::connect(address).await.unwrap();
|
|
client
|
|
.write_all(
|
|
b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: Upgrade\r\nUpgrade: h2c\r\n\r\n",
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let response = String::from_utf8(read_http1_head(&mut client).await).unwrap();
|
|
assert!(response.starts_with("HTTP/1.1 400"), "{response}");
|
|
stop.cancel();
|
|
server.await.unwrap();
|
|
}
|
|
}
|