From 586ff8f223ff5729f554554a406d394ebe53d38e Mon Sep 17 00:00:00 2001 From: Hector van der Aa Date: Tue, 15 Sep 2026 10:57:11 +0200 Subject: [PATCH] checkpoint: integrate authenticated transports --- Cargo.lock | 105 +- Cargo.toml | 10 +- README.md | 121 +-- docs/ARCHITECTURE.md | 16 +- docs/DEVELOPMENT_STATE.md | 186 ++-- docs/IMPLEMENTATION_CHECKLIST.md | 20 +- docs/OPERATIONS.md | 116 ++- docs/PROTOCOL.md | 36 +- docs/TROUBLESHOOTING.md | 30 +- examples/client.advanced.toml | 4 +- examples/server.advanced.toml | 4 +- examples/server.minimal.toml | 10 +- packaging/systemd/wgcloak-client.service | 2 +- packaging/systemd/wgcloak-server.service | 2 +- src/cli.rs | 6 +- src/client/mod.rs | 457 +++++++- src/client/supervisor.rs | 41 +- src/config.rs | 541 +++++++++- src/control.rs | 349 ++++++- src/lib.rs | 12 + src/main.rs | 22 +- src/protocol/auth.rs | 4 + src/protocol/direct.rs | 82 +- src/server/limits.rs | 3 + src/server/mod.rs | 603 ++++++++++- src/stats.rs | 87 +- src/transport/direct_udp.rs | 440 ++++++-- src/transport/http3.rs | 1208 +++++++++++++++++++++- src/transport/mod.rs | 637 ++++++++++-- src/transport/quic.rs | 142 ++- tests/control.rs | 73 +- tests/direct_udp.rs | 335 +++++- tests/http3.rs | 39 +- tests/http3_integration.rs | 272 +++++ tests/http3_runtime.rs | 394 +++++++ tests/integration_transports.rs | 113 +- tests/limits.rs | 141 +++ tests/quic.rs | 48 +- tests/reconnect.rs | 161 +++ tests/runtime_lifecycle.rs | 146 +++ tests/supervisor.rs | 22 +- tests/tls_tcp.rs | 8 +- 42 files changed, 6277 insertions(+), 771 deletions(-) create mode 100644 src/lib.rs create mode 100644 src/server/limits.rs create mode 100644 tests/http3_integration.rs create mode 100644 tests/http3_runtime.rs create mode 100644 tests/limits.rs create mode 100644 tests/reconnect.rs create mode 100644 tests/runtime_lifecycle.rs diff --git a/Cargo.lock b/Cargo.lock index c167a57..d1d2bbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,29 +67,6 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" -[[package]] -name = "aws-lc-rs" -version = "1.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", - "pkg-config", -] - [[package]] name = "base64" version = "0.22.1" @@ -130,8 +107,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3eb0f42d6c360dc3f8a821f6bf2fdea7f72bfd36b3076eb0e6d1e9e0752fff4" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] @@ -198,15 +173,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] - [[package]] name = "colorchoice" version = "1.0.5" @@ -278,12 +244,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "equivalent" version = "1.0.2" @@ -300,18 +260,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "fastbloom" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" -dependencies = [ - "foldhash", - "libm", - "portable-atomic", - "siphasher", -] - [[package]] name = "fastrand" version = "2.5.0" @@ -324,18 +272,6 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.34" @@ -618,16 +554,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "jobserver" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" -dependencies = [ - "getrandom 0.4.3", - "libc", -] - [[package]] name = "js-sys" version = "0.3.105" @@ -651,12 +577,6 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "libm" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -766,18 +686,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pkg-config" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" - -[[package]] -name = "portable-atomic" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -824,7 +732,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" dependencies = [ "bytes", - "fastbloom", "getrandom 0.4.3", "lru-slab", "rand 0.10.2", @@ -1000,8 +907,6 @@ version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ - "aws-lc-rs", - "log", "once_cell", "ring", "rustls-pki-types", @@ -1065,7 +970,6 @@ version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -1230,12 +1134,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" -[[package]] -name = "siphasher" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" - [[package]] name = "slab" version = "0.4.12" @@ -1470,7 +1368,6 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -1651,10 +1548,12 @@ dependencies = [ "bytes", "clap", "h3", + "h3-datagram", "h3-quinn", "hex", "hkdf", "hmac", + "http", "quinn", "rand 0.8.8", "rustls", diff --git a/Cargo.toml b/Cargo.toml index c097f50..fb4e26f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "wgcloak" version = "0.1.0" edition = "2021" description = "Protocol-agnostic authenticated UDP transport layer" -license = "MIT OR Apache-2.0" +license = "MIT" [dependencies] anyhow = "1.0" @@ -12,12 +12,14 @@ base64 = "0.22" clap = { version = "4.5", features = ["derive"] } h3 = "0.0.8" h3-quinn = { version = "0.0.10", features = ["datagram"] } +h3-datagram = "0.0.2" +http = "1" hkdf = "0.12" hmac = "0.12" hex = "0.4" -quinn = "0.11" +quinn = { version = "0.11", default-features = false, features = ["runtime-tokio", "rustls-ring", "platform-verifier"] } rand = "0.8" -rustls = "0.23" +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } secrecy = "0.8" serde = { version = "1.0", features = ["derive"] } sha2 = "0.10" @@ -26,7 +28,7 @@ subtle = "2.6" thiserror = "1.0" tokio = { version = "1.40", features = ["full"] } tokio-util = { version = "0.7", features = ["codec", "rt"] } -tokio-rustls = "0.26" +tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } rustls-native-certs = "0.8" toml = "0.8" tracing = "0.1" diff --git a/README.md b/README.md index 3f4bec4..0f5126d 100644 --- a/README.md +++ b/README.md @@ -1,98 +1,99 @@ # wgcloak -wgcloak is a Linux userspace relay for opaque UDP datagrams. It is intended to -carry WireGuard traffic, but does not configure, inspect, or depend on -WireGuard: the same relay can carry any UDP application. +wgcloak relays opaque UDP datagrams between a local UDP application and a +remote UDP backend. It does not inspect WireGuard or other application +protocols and does not configure interfaces, routes, or firewalls. -The design keeps four concerns separate: - -```text -local UDP application -> tunnel core -> selected transport -> tunnel core - -> backend UDP application -``` - -The server gives each authenticated client its own backend UDP socket. The -application payload is not decoded or rewritten. - -## Status - -This checkout is specification-first and under active implementation. The -milestone and v1 acceptance checklist is in -[`docs/IMPLEMENTATION_CHECKLIST.md`](docs/IMPLEMENTATION_CHECKLIST.md). A -transport is available only when its code, tests, and integration criteria are -verified; the documentation intentionally does not imply that all four -transports are already present. - -Target v1 transport order: - -1. direct UDP (Milestone 1) -2. QUIC DATAGRAM (Milestone 2) -3. HTTP/3 CONNECT-UDP (Milestone 4) -4. TLS/TCP framed fallback (Milestone 5) +The current runtime implements authenticated direct UDP, QUIC DATAGRAM, +HTTP/3 CONNECT-UDP, and TLS/TCP framed fallback. Client `mode = "auto"` probes +the configured transport order; authentication rejection stops fallback. +See [`docs/IMPLEMENTATION_CHECKLIST.md`](docs/IMPLEMENTATION_CHECKLIST.md) for +acceptance evidence and remaining v1 work. ## Build and install -Rust and Cargo are required. From the repository root: +Install Rust and Cargo, then run from the repository root: ```bash -cargo test +cargo test --all-targets cargo build --release ``` -For a host installation (choose a reviewed release artifact): +For a local host installation: ```bash sudo install -Dm755 target/release/wgcloak /usr/local/bin/wgcloak -sudo install -d -m0750 /etc/wgcloak +sudo install -d -o root -g wgcloak -m0750 /etc/wgcloak/clients ``` -The binary name and subcommands are the intended interface from `spec.md`: +The CLI supports: ```bash wgcloak client --config /etc/wgcloak/client.toml wgcloak server --config /etc/wgcloak/server.toml wgcloak check-config --config /etc/wgcloak/client.toml wgcloak generate-secret +wgcloak status +wgcloak stats +wgcloak reconnect +wgcloak reload +wgcloak shutdown ``` -Run `wgcloak --help` against the built binary to confirm the commands in the -current build. QUIC, HTTP/3, control commands, and certificate reload are -milestone-gated; see the checklist before deploying them. +`reload` is retained as a CLI/control command but currently returns an explicit +unsupported response. Configuration and certificate hot reload are not +implemented. `reconnect` applies to the client; the server rejects it. -## Configuration +## Configure and run -Copy and edit one of the safe, placeholder-only examples: +Copy a placeholder-only example and adjust it for the selected transport: - [`examples/client.minimal.toml`](examples/client.minimal.toml) - [`examples/client.advanced.toml`](examples/client.advanced.toml) - [`examples/server.minimal.toml`](examples/server.minimal.toml) - [`examples/server.advanced.toml`](examples/server.advanced.toml) -Put real secrets in separate files, owned by the service account, mode `0600`. -Do not commit them. See [`docs/OPERATIONS.md`](docs/OPERATIONS.md) and -[`docs/SECURITY.md`](docs/SECURITY.md). +Store each secret in a separate file owned by the service account with mode +`0600`. Keep configuration and key files out of version control. TLS-capable +transports require a certificate and private key on the server; clients use +normal certificate-chain and hostname verification. -## Run with systemd +Run in the foreground first: -The templates in [`packaging/systemd/`](packaging/systemd/) assume a dedicated +```bash +wgcloak check-config --config /etc/wgcloak/client.toml +wgcloak client --config /etc/wgcloak/client.toml +``` + +See [`docs/OPERATIONS.md`](docs/OPERATIONS.md) for control commands, systemd +deployment, shutdown behavior, and troubleshooting pointers. + +## systemd + +The units in [`packaging/systemd/`](packaging/systemd/) assume a dedicated `wgcloak` user, `/usr/bin/wgcloak`, and configuration under `/etc/wgcloak/`. -Review paths and transport support before installing them. The server template -includes hardening and a commented optional `CAP_NET_BIND_SERVICE` setting for -ports below 1024; normal operation does not require `CAP_NET_ADMIN` or -`CAP_SYS_ADMIN`. +Review the paths and enabled listeners before installing. The server unit +contains an optional, commented `CAP_NET_BIND_SERVICE` setting for ports below +1024. Normal operation needs neither `CAP_NET_ADMIN` nor `CAP_SYS_ADMIN`. + +## Security boundaries + +TLS transports authenticate the server with normal X.509 verification and +authenticate clients with the configured shared secret. Direct UDP provides +message integrity and replay protection but no outer confidentiality. The +control socket is local, stored in a private runtime directory, and mode +`0600`. + +wgcloak does not configure WireGuard, create TUN/TAP devices, change routes or +firewalls, fragment application datagrams, spoof browser fingerprints, +cluster servers, or provide multiple simultaneous local client tunnels. ## Documentation -- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md): process, data path, and ownership boundaries. -- [`docs/PROTOCOL.md`](docs/PROTOCOL.md): direct packet model and transport contracts. -- [`docs/OPERATIONS.md`](docs/OPERATIONS.md): files, control socket, lifecycle, and deployment. -- [`docs/SECURITY.md`](docs/SECURITY.md): threat model, secrets, privileges, and non-goals. -- [`docs/TROUBLESHOOTING.md`](docs/TROUBLESHOOTING.md): checks for common startup and connectivity failures. -- [`docs/IMPLEMENTATION_CHECKLIST.md`](docs/IMPLEMENTATION_CHECKLIST.md): milestones 1–6 and all 17 v1 criteria. - -## Explicit non-goals - -wgcloak does not configure WireGuard, create TUN/TAP devices, change routes or -firewalls, fragment application datagrams, spoof browser/TLS fingerprints, or -provide clustering, roaming, a GUI, Windows/macOS support, or multiple client -tunnels in v1. +- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md): runtime roles and ownership. +- [`docs/PROTOCOL.md`](docs/PROTOCOL.md): wire protocol and transport behavior. +- [`docs/OPERATIONS.md`](docs/OPERATIONS.md): setup, control, and systemd. +- [`docs/SECURITY.md`](docs/SECURITY.md): threat model and permissions. +- [`docs/TROUBLESHOOTING.md`](docs/TROUBLESHOOTING.md): common failures. +- [`docs/IMPLEMENTATION_CHECKLIST.md`](docs/IMPLEMENTATION_CHECKLIST.md): + milestones 1–6 and all 17 v1 criteria. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c639efc..e67671d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -49,10 +49,12 @@ STARTING → RESOLVING → PROBING → AUTHENTICATING → CONNECTED └→ PROBING ``` -In `auto` mode the target order is direct UDP, QUIC DATAGRAM, HTTP/3 -CONNECT-UDP, then TLS/TCP. Authentication rejection is a configuration error, -not permission to probe indefinitely. Network failures use bounded exponential -backoff with jitter. +In `auto` mode the configured order defaults to direct UDP, QUIC DATAGRAM, +HTTP/3 CONNECT-UDP, then TLS/TCP. Authentication rejection is a configuration +error, not permission to probe indefinitely. Network failures use bounded +exponential backoff with jitter (250 ms to 30 s), reset after a successful +connection. These transports are wired into the current runtime; the +acceptance checklist distinguishes implementation from completed v1 sign-off. ## Server isolation @@ -69,5 +71,7 @@ Normal operation uses ordinary UDP/TCP/Unix sockets and needs no root, reviewed deployment must bind a port below 1024. The control socket is local and protected by filesystem ownership/mode; it is not a network listener. -This document describes the intended v1 architecture. Check the milestone -matrix before relying on a transport or control command in a particular build. +Configuration reload and certificate/key reload are not implemented. The +control `reload` command returns an explicit unsupported response. The +`reconnect` command applies to the client, while the server supports `status`, +`stats`, and graceful `shutdown`. diff --git a/docs/DEVELOPMENT_STATE.md b/docs/DEVELOPMENT_STATE.md index 027cbe0..7bae75b 100644 --- a/docs/DEVELOPMENT_STATE.md +++ b/docs/DEVELOPMENT_STATE.md @@ -1,109 +1,113 @@ # Development state -Checkpoint: 2026-09-14, end of day. Development is paused at a compilable -checkpoint; no commit was made. +End-of-day checkpoint: 2026-09-15. Branch: `main`. Baseline commit: +`313ab00` (`checkpoint: implement wgcloak v1 foundations`). Work after that +checkpoint remains uncommitted in the shared worktree. Several agents have +edited files in parallel; use the inventory below and review the complete diff +before committing. -## Branch and worktree +## Implemented behavior -- Branch: `main`. -- `git status --short` reports the project files as untracked, including - `Cargo.toml`, `Cargo.lock`, `src/`, `tests/`, documentation, examples, and - packaging files. There is no useful committed baseline to compare against in - this checkout. -- Multiple agents have been working in the shared directory. Review all diffs - and ownership boundaries before committing; in particular, do not assume an - untracked file was created by this task or that a concurrent edit is final. +- The package now has a reusable Rust library (`src/lib.rs`) and a thin binary + (`src/main.rs`). Integration tests import `wgcloak` instead of compiling + production modules through `#[path]`; module unit tests run once in the + library test target. +- Client and server runtime paths support authenticated direct UDP, QUIC + DATAGRAM, HTTP/3 CONNECT-UDP with HTTP Datagrams, and TLS/TCP length-framed + fallback. TLS client connections use normal certificate-chain and hostname + verification. For encrypted transports, shared-secret authentication occurs + before server backend-session activation. +- Client `auto` uses its configured transport order and bounded connection + probes, stops on authentication rejection, and reconnects with jittered + exponential backoff from 250 ms through 30 s. Successful connections reset + the backoff. The client control `reconnect` command forces another attempt. +- The server runs enabled listeners concurrently, rejects incompatible UDP + listener port conflicts, and aggregates listener/session statistics. Sessions + are isolated and use per-session backend UDP sockets. Admission limits cover + active sessions, per-IP sessions, pending handshakes, and authentication + attempts. +- The Unix control socket uses a private runtime directory and mode `0600`, + refuses an active socket, and removes only a verified stale socket. `status`, + `stats`, and `shutdown` are supported; `reconnect` is client-only. Config and + certificate/key reload are not implemented. The CLI/control reload request + returns an explicit unsupported response. +- The local client UDP listener currently tracks one most-recent application + peer; concurrent local source peers can replace the return target. +- Direct UDP keepalives are one-way. Silent peer loss may not be detected + immediately; a local transport error or explicit client reconnect is the + recovery path covered by tests. -## Implemented and wired behavior +## Verification checkpoint -The current tree contains the following work, subject to integration review: +Before the final config and client-ID edits, the full gates passed: -- Operational assets: MIT `LICENSE`, Linux-focused `README.md`, `.gitignore`, - client/server TOML examples, architecture/protocol/security/operations/ - troubleshooting documents, an implementation checklist, and systemd client - and server templates. -- TLS/TCP transport primitives in `src/transport/tls_tcp.rs`: bounded - incremental/coalesced u16-big-endian framing, truncated/oversized-frame - handling, PEM/DER certificate and private-key loading, challenge-response - authentication, authenticated data/heartbeat/close framing, idle timeout, - and bounded shutdown. -- HTTP/3 support primitives in `src/transport/http3.rs`: CONNECT-UDP request - validation, QUIC varints, RFC 9297 HTTP Datagram context encoding, capsule - encoding, and project authentication messages. The module does not fake - CONNECT-UDP with POST or ordinary streams. -- Runtime orchestration in `src/transport/mod.rs`, `src/client/mod.rs`, and - `src/server/mod.rs` instantiates direct UDP, QUIC, and TLS/TCP connectors or - listeners. TLS backend creation occurs after the authenticated session is - established. HTTP/3 currently returns an explicit `Unsupported` result from - runtime selection because the actual h3 datagram adapter is not complete. -- The client and server start the Unix control server. Client reconnect and - reload commands currently cause a reconnect attempt; server shutdown is - handled by the encrypted listener paths, while direct-UDP/control lifecycle - still needs review. -- `h3-quinn` was enabled with its `datagram` feature; `tokio-rustls` and - `rustls-native-certs` were added for TLS runtime integration. +- `cargo fmt --all -- --check` +- `cargo check --all-targets` +- `cargo clippy --all-targets -- -D warnings` +- `cargo test --all-targets --quiet` — 76 tests passed. -## Files changed in this work +After those edits, the config-focused suite passed 12/12, the client-ID focused +transport tests passed, and the current `cargo check` passes. Rerun the full +gates after these edits before treating the tree as audited. The broader +encrypted daemon-runtime end-to-end proof described below is not yet created or +complete; focused transport and HTTP/3 runtime tests do not replace it. -Primary files touched by the operations/integration work are: +## Remaining work, prioritized -- `Cargo.toml`, `Cargo.lock` -- `src/transport/mod.rs` -- `src/transport/tls_tcp.rs` -- `src/transport/http3.rs` -- `src/client/mod.rs` -- `src/server/mod.rs` -- `tests/tls_tcp.rs`, `tests/http3.rs`, `tests/integration_transports.rs` -- `.gitignore`, `LICENSE`, `README.md` -- `examples/*`, `packaging/systemd/*`, and the other files under `docs/` +1. Add and complete encrypted daemon-runtime end-to-end proof: local UDP + application traffic through client runtime, encrypted transport, server + runtime, and UDP echo backend, including auth rejection, session isolation, + status/stats, and bounded shutdown. Exercise each encrypted mode where + practical; do not mistake lower-level transport tests for daemon proof. +2. Rerun formatting, all-target check, strict all-target Clippy, and the full + suite after the final config/client-ID edits; then review the complete diff + and reconcile acceptance evidence with `docs/IMPLEMENTATION_CHECKLIST.md`. +3. Add fuzz and benchmark targets for parsers, framing, protocol/auth paths, + and transport overhead; record the environment and baseline. +4. Decide and document a practical maximum datagram-size cap, with config and + boundary tests tied to transport overhead and MTU constraints. +5. Confirm whether configuration and certificate/key reload are required by + the spec/product scope. Until implemented and tested, they remain explicitly + unsupported; do not imply hot reload in CLI or deployment docs. -`src/transport/quic.rs` was edited by another concurrent agent and is outside -the TLS/integration ownership listed above. Preserve that ownership boundary -when resolving conflicts. +## First recovery tasks -## Verification and current failures +1. Inspect current client/server orchestration and transport test APIs, then + create the encrypted daemon-runtime echo test without duplicating transport + internals. +2. Re-run the four full gates above against the current tree; resolve any + compile, lint, or test regressions before adding further features. +3. Review spec milestones and the 17 completion criteria, and record evidence + for each criterion in the coordinator checklist. Keep fuzzing, benchmarks, + datagram-cap decision, and reload scope visible as open items. -- TOML examples were parsed successfully with Python `tomllib`. -- TLS/TCP focused tests passed: `cargo test --test tls_tcp --quiet` (7 tests). -- HTTP/3 focused tests passed: `cargo test --test http3 --quiet` (4 tests). -- `cargo check` passed after the transport integration edits. -- The TLS loopback integration test compiles but currently panics while - constructing a rustls configuration because no process-level crypto provider - is installed: +## Current worktree inventory - `Could not automatically determine process-level CryptoProvider from Rustls crate features.` +Captured with `git status --short` for this checkpoint. These are shared-tree +changes; ownership is not implied by this list. - The runtime/test setup must explicitly install the `ring` provider (or enable - exactly one provider feature) before building rustls configurations. -- The supervisor's latest failure reprobe reports that the QUIC focused tests - now pass. Run the complete suite again after fixing the rustls provider; the - earlier complete-suite run had QUIC handshake-timeout failures before the - concurrent QUIC-agent fixes landed. -- Clippy has not reached a clean result. Existing included test/source modules - produce many dead-code and lint diagnostics, and integration code also needs - a final lint pass after the provider fix. +Modified files: -## Remaining gaps, prioritized +- `Cargo.lock`, `Cargo.toml` +- `README.md` +- `docs/ARCHITECTURE.md`, `docs/DEVELOPMENT_STATE.md`, + `docs/IMPLEMENTATION_CHECKLIST.md`, `docs/OPERATIONS.md`, `docs/PROTOCOL.md`, + `docs/TROUBLESHOOTING.md` +- `examples/client.advanced.toml`, `examples/server.advanced.toml`, + `examples/server.minimal.toml` +- `packaging/systemd/wgcloak-client.service`, + `packaging/systemd/wgcloak-server.service` +- `src/cli.rs`, `src/client/mod.rs`, `src/client/supervisor.rs`, `src/config.rs`, + `src/control.rs`, `src/main.rs`, `src/protocol/auth.rs`, + `src/protocol/direct.rs`, `src/server/mod.rs`, `src/stats.rs`, + `src/transport/direct_udp.rs`, `src/transport/http3.rs`, + `src/transport/mod.rs`, `src/transport/quic.rs` +- `tests/control.rs`, `tests/direct_udp.rs`, `tests/http3.rs`, + `tests/integration_transports.rs`, `tests/quic.rs`, `tests/supervisor.rs`, + `tests/tls_tcp.rs` -1. Fix and verify explicit rustls crypto-provider initialization, then run the - TLS loopback test and the complete test suite. -2. Finish a real h3 0.0.8/h3-quinn 0.0.10 HTTP/3 CONNECT-UDP datagram adapter, - or retain the explicit `Unsupported` path with a precise documented - dependency/API blocker. Do not substitute POST, capsules-only streams, or - ordinary HTTP request bodies for HTTP Datagrams. -3. Review runtime lifecycle semantics: direct-UDP control shutdown wiring, - actual file-backed config reload, client TLS heartbeat scheduling, and - authentication-error mapping so auto fallback stops on rejected credentials. -4. Run repository-wide formatting, tests, and clippy; inspect the shared - untracked tree and reconcile concurrent `quic.rs` changes before any commit. - -## First three tasks tomorrow - -1. Install/select the rustls `ring` provider once at runtime and in the - integration test setup; rerun TLS loopback plus `cargo test`. -2. Inspect the enabled h3-quinn datagram API and implement/test the smallest - standards-correct CONNECT-UDP client/server adapter; otherwise record the - exact unsupported API/dependency condition in the public docs. -3. Perform an end-to-end lifecycle review and repository-wide `fmt`, check, - test, and clippy pass, resolving only changes within each agent's ownership. +Untracked files: +- `src/lib.rs`, `src/server/limits.rs` +- `tests/http3_integration.rs`, `tests/http3_runtime.rs`, `tests/limits.rs`, + `tests/reconnect.rs`, `tests/runtime_lifecycle.rs` diff --git a/docs/IMPLEMENTATION_CHECKLIST.md b/docs/IMPLEMENTATION_CHECKLIST.md index ec34202..3cd851b 100644 --- a/docs/IMPLEMENTATION_CHECKLIST.md +++ b/docs/IMPLEMENTATION_CHECKLIST.md @@ -1,9 +1,10 @@ # Implementation and acceptance checklist -This is a coordinator checklist, not a claim that the items are complete. -Mark an item only with code evidence, tests, and an integration result. Record -the commit/test command or issue beside each checked item. A feature marked -planned in the other documents must remain visibly planned here until proven. +This is a coordinator checklist, not a claim that v1 is signed off. The current +runtime has code paths for all four carriers, configured auto selection, +client reconnect backoff, and local control commands. Mark an acceptance item +only with code evidence, tests, and an integration result; record the test +command or issue beside each checked item. ## Milestones 1–6 @@ -14,17 +15,18 @@ planned in the other documents must remain visibly planned here until proven. - [ ] **2 — QUIC DATAGRAM:** certificate loading and hostname verification; client challenge/response; datagrams; keepalive/MTU; reconnect; multi-client handling without core API redesign. -- [ ] **3 — automatic selection:** `mode = "auto"`; direct UDP then QUIC; - bounded probes/retries; auth rejection stops probing; jittered reconnect; - status reports selected transport. +- [ ] **3 — automatic selection:** `mode = "auto"`; configured order across + direct UDP, QUIC, HTTP/3, and TLS/TCP; bounded probes/retries; auth rejection + stops probing; jittered reconnect; status reports selected transport. - [ ] **4 — HTTP/3 CONNECT-UDP:** standard CONNECT-UDP/HTTP Datagram semantics; authentication; backend mapping; reuse QUIC infrastructure where practical. - [ ] **5 — TLS/TCP fallback:** TLS listener; client authentication; bounded big-endian length framing; bidirectional datagram reconstruction; fallback integration; expected head-of-line behavior documented. - [ ] **6 — operational polish:** Unix control socket; status/stats/reconnect - commands; config validation; systemd units; capability guidance; certificate - and configuration reload; benchmarks; fuzzing; installation docs. + commands; config validation; systemd units; capability guidance; benchmarks; + fuzzing; installation docs. Config and certificate/key reload are not + implemented; the control command explicitly reports reload as unsupported. ## v1 completion criteria (spec §89) diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index b8018fc..bbf2617 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -1,79 +1,91 @@ # Operations -This guide assumes a dedicated `wgcloak` system user and `/etc/wgcloak/` for -configuration. Adjust paths for a user service or a different packaging -layout. +This guide assumes a dedicated `wgcloak` service account and configuration +under `/etc/wgcloak/`. Adjust paths to match the reviewed deployment. -## First installation +## Build and install ```bash +cargo test --all-targets +cargo build --release sudo install -Dm755 target/release/wgcloak /usr/bin/wgcloak sudo install -d -o wgcloak -g wgcloak -m0750 /etc/wgcloak/clients -sudo install -o wgcloak -g wgcloak -m0600 client.secret /etc/wgcloak/client.secret ``` -Create server client secrets separately and transfer them through a protected -channel. Do not put real secret contents in the repository. Validate a config -before starting a service when `check-config` is available in the current -build: +Store each client secret in a separate file readable only by the service +account (normally owner `wgcloak`, mode `0600`). Protect the parent directory +from untrusted writes. The server needs its TLS certificate and private key +readable by its service account when QUIC, HTTP/3, or TLS/TCP is enabled. + +Validate configuration before starting a daemon: ```bash sudo -u wgcloak wgcloak check-config --config /etc/wgcloak/client.toml +sudo -u wgcloak wgcloak check-config --config /etc/wgcloak/server.toml ``` -The command is a target v1 CLI and is delivered in the operational-polish -milestone; confirm with `wgcloak --help`. +## Start, stop, and reconnect -## Starting and stopping - -Foreground operation is useful for first-run diagnostics: +Foreground operation is useful during initial setup: ```bash wgcloak client --config /etc/wgcloak/client.toml wgcloak server --config /etc/wgcloak/server.toml ``` -SIGTERM and SIGINT should stop accepting new sessions, stop local ingestion, -close backend sockets, flush useful metrics, and exit within a bounded timeout. -A transport failure should reconnect with jitter; it should not terminate the -daemon. The systemd templates set restart-on-failure for this reason. +On Linux, SIGINT, SIGTERM, or the control `shutdown` command initiates bounded +shutdown. The server stops accepting sessions and gives listeners/sessions a +bounded cleanup window before abort fallback. A client reconnects after +transport failures using exponential backoff with jitter from 250 ms to 30 s; +the sequence resets after a successful connection. Authentication rejection +stops automatic probing rather than trying other transports. -## Control socket (intended interface) +Direct UDP keepalives are one-way and do not acknowledge peer liveness. A +silent direct-UDP server restart may therefore not be detected immediately; +use `wgcloak reconnect` on the client to force a fresh handshake. The tested +restart path waits through normal reconnect backoff while the server is +unavailable, then resumes when it returns. -The daemon should expose a Unix-domain control socket at -`$XDG_RUNTIME_DIR/wgcloak.sock` (for a system service, the template maps this -to its private runtime directory). Filesystem permissions, not a TCP port, -authorize access. The initial control protocol may be newline-delimited JSON. +## Local control socket -Target commands are: +By default the socket is `$XDG_RUNTIME_DIR/wgcloak.sock` when that runtime +directory is owned by the current user and has mode `0700`. Otherwise the +daemon uses `/run/user//wgcloak/wgcloak.sock` when that private user +runtime directory exists, or `$HOME/.local/run/wgcloak/wgcloak.sock` as a +per-user fallback. The socket's parent is created with mode `0700`; the socket +itself is mode `0600`. An active socket path is never unlinked; a stale socket +is removed only after its type and identity are checked. Use the same runtime +directory for the CLI that is used by the daemon, or pass `--socket` +explicitly (its parent directory must be owned by you and mode `0700`): -```text -status current role/state, selected transport, endpoint, MTU and uptime -stats counters for traffic, drops, auth failures, reconnects and sessions -reconnect ask the client to leave the current transport and probe again -reload re-read supported configuration (not full hot reload in early builds) -shutdown request graceful process shutdown +```bash +wgcloak status +wgcloak stats +wgcloak reconnect +wgcloak reload +wgcloak shutdown ``` -The exact request/response field names are implementation-owned until the -control protocol lands. A client should never send credentials or payloads -through this socket. `status` and `stats` are read-only; `reconnect`, `reload`, -and `shutdown` are state-changing and must remain local-admin operations. +`status` and `stats` report the running state. `reconnect` asks a client to +leave its current session and reconnect; the server rejects this client-only +command. `reload` currently returns “configuration reload is not supported”. +Neither configuration hot reload nor certificate/key reload is implemented. +`shutdown` requests graceful process shutdown. The control protocol is local +newline-delimited JSON; do not send credentials or application payloads over +it. + +For a non-default control path, set `XDG_RUNTIME_DIR` consistently for both +daemon and CLI, or use the CLI's `--socket PATH` option. ## systemd deployment -Review and copy the units under `packaging/systemd/`. They assume: - -- binary `/usr/bin/wgcloak`; -- user/group `wgcloak`; -- configuration `/etc/wgcloak/client.toml` or `server.toml`; -- runtime socket under a private `/run` directory; -- secret and certificate paths readable by the service user. - -After review: +Review and copy the units under `packaging/systemd/`. They assume binary +`/usr/bin/wgcloak`, user/group `wgcloak`, and role configuration under +`/etc/wgcloak/`. Each unit creates a private runtime directory and sets +`XDG_RUNTIME_DIR` there. ```bash -sudo install -Dm0644 packaging/systemd/wgcloak-client.service /etc/systemd/system/wgcloak-client.service +sudo install -Dm0644 packaging/systemd/wgcloak-client.service /etc/systemd/system/ sudo systemctl daemon-reload sudo systemctl enable --now wgcloak-client.service systemctl status wgcloak-client.service @@ -81,13 +93,15 @@ journalctl -u wgcloak-client.service -f ``` For a low port such as 443, uncomment `AmbientCapabilities=CAP_NET_BIND_SERVICE` -and its bounding-set companion in the server unit only after confirming the -unit still starts. Direct UDP examples use an unprivileged port. +and its matching bounding-set entry in the server unit only if required. Normal +high-port operation needs no capability. Keep `CAP_NET_ADMIN` and +`CAP_SYS_ADMIN` disabled. ## Safe rollout -Start with direct UDP and one test client. Check local listener binding, -server authentication, backend reachability, return-peer behavior, counters, -and clean shutdown. Add one transport milestone at a time. Keep an independent -path to the host while changing a public listener and retain the previous -configuration until a restart test succeeds. +Start with one enabled transport and one test client. Verify authentication, +backend reachability, reply delivery, status/statistics, and clean shutdown. +Add transports one at a time. Direct UDP, QUIC, and HTTP/3 are UDP listeners; +QUIC and HTTP/3 cannot bind the same address/port in the current architecture. +TLS/TCP may use the same numeric port because it uses TCP. Keep an independent +management path available when changing public listener addresses. diff --git a/docs/PROTOCOL.md b/docs/PROTOCOL.md index e040822..db15b88 100644 --- a/docs/PROTOCOL.md +++ b/docs/PROTOCOL.md @@ -49,20 +49,30 @@ starting limit is 1200 bytes. ## Encapsulated transports -- **Direct UDP:** lowest overhead and native datagrams; uses the authenticated - protocol above and provides integrity but not outer confidentiality. -- **QUIC DATAGRAM:** TLS 1.3 protects the transport; client shared-secret - challenge/response still authenticates the configured client. -- **HTTP/3 CONNECT-UDP:** uses legitimate MASQUE CONNECT-UDP and HTTP Datagram - semantics over QUIC. It is for compatibility, not fingerprint spoofing. -- **TLS/TCP:** uses the same client authentication model. Each datagram is - framed as a big-endian `u16` payload length followed by up to 65535 bytes; - the configured lower limit should normally apply. TCP head-of-line blocking - is expected. +- **Direct UDP:** native datagrams; uses the authenticated protocol above and + provides integrity but not outer confidentiality. +- **QUIC DATAGRAM:** TLS 1.3 protects the connection; a client shared-secret + challenge/response authenticates the configured client. Application + datagrams use QUIC DATAGRAM frames. +- **HTTP/3 CONNECT-UDP:** uses extended CONNECT with `:protocol=connect-udp` + and HTTP Datagrams with the RFC 9297 context ID. The request path is + configurable. Capsules are used for the authentication exchange, not as the + application-data carrier. +- **TLS/TCP:** uses the same client authentication model. Each application + datagram is framed by a big-endian `u16` payload length and opaque payload; + the configured maximum applies. TCP head-of-line blocking is expected. -QUIC, HTTP/3, and TLS/TCP are planned by milestones 2, 4, and 5 respectively. -The direct packet details are a protocol target; confirm the implementation and -integration tests before treating them as interoperable guarantees. +All four carriers are wired into the current client/server runtime. Client +`mode = "auto"` probes its configured order, retries bounded probe attempts, +and stops on authentication rejection. Reconnection after transport loss uses +bounded exponential backoff with jitter; successful connection resets the +backoff. This describes the current code path, not a claim that every v1 +acceptance criterion has passed. + +The control `reload` command is currently unsupported, including certificate +or key reload. See the checklist for acceptance work that remains before a v1 +release. The direct packet details and the encapsulated transport APIs are +project-specific; interoperability requires matching this protocol. ## Authentication and resource order diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 699ffc3..570691e 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -8,15 +8,16 @@ Check the simplest layer that can explain the symptom, then move outward. error. - Check that every secret/certificate path exists and is readable by the service account, but not by unrelated users. -- Run `wgcloak --help` and, where implemented, `wgcloak check-config` to spot a - command or schema mismatch. +- Run `wgcloak --help` and `wgcloak check-config` to spot a command or schema + mismatch. - Confirm listener addresses are valid and no other process owns the port. ## Client cannot connect - Verify DNS resolution and the configured server name/port. -- Confirm the selected transport is implemented in this build; an `auto` list - may mention future transports. +- Confirm the selected server listener is enabled and that the matching + transport/port appears in its `status` output. In auto mode, check the + configured client order and `retry_ms` status while it probes. - For direct UDP, check both directions in the host firewall and allow the server's UDP listener. - For TLS transports, verify the hostname matches the certificate and do not @@ -44,17 +45,22 @@ fallback probing rather than generate traffic across every transport. ## Session disappears -The target idle timeout is 120 seconds, refreshed by authenticated data or -keepalive. Check keepalive settings, NAT expiry, server resource limits, and -whether the client is reconnecting after a transport failure. A valid direct -UDP packet can update the peer address after NAT rebinding; a spoofed packet -without a valid MAC cannot. +The server's configured idle timeout applies per session. Check transport +keepalive behavior, NAT expiry, server resource limits, and whether the client +is reconnecting after a transport failure. A valid direct UDP packet can +update the peer address after NAT rebinding; a spoofed packet without a valid +MAC cannot. ## Control socket unavailable -Check the daemon's `XDG_RUNTIME_DIR`, socket ownership/mode, and whether a -stale socket path remains after an unclean exit. The control interface is local -only and may be absent until the operational-polish milestone is implemented. +Check the daemon and CLI use the same `XDG_RUNTIME_DIR`, and verify the +directory is private and the socket mode is `0600`. If `XDG_RUNTIME_DIR` is +unset or is not current-user-owned mode `0700`, the daemon uses +`/run/user//wgcloak/` when available, or `$HOME/.local/run/wgcloak/`. +An active socket is never removed; a stale socket can be replaced only after +its type and identity are checked. +The `reload` command is present but currently returns an unsupported response; +neither configuration nor certificate hot reload is implemented. ## Host hardening prevents startup diff --git a/examples/client.advanced.toml b/examples/client.advanced.toml index a75a9bd..5c755fd 100644 --- a/examples/client.advanced.toml +++ b/examples/client.advanced.toml @@ -11,7 +11,7 @@ listen = "127.0.0.1:51821" [transport] mode = "auto" -# Auto probing is target behavior; use only transports supported by the build. +# Auto probing follows this order; authentication rejection stops fallback. [transport.auto] order = ["direct_udp", "quic", "http3", "tls_tcp"] probe_timeout_ms = 1500 @@ -26,7 +26,7 @@ port = 443 keepalive_secs = 15 [transport.http3] -port = 443 +port = 4443 path = "/tunnel" [transport.tls_tcp] diff --git a/examples/server.advanced.toml b/examples/server.advanced.toml index ca9b874..7a6a876 100644 --- a/examples/server.advanced.toml +++ b/examples/server.advanced.toml @@ -1,7 +1,7 @@ # Advanced server example. Values are placeholders, not credentials. [server] -listen = "0.0.0.0:51830" +listen = "0.0.0.0" [backend] target = "127.0.0.1:51820" @@ -16,7 +16,7 @@ port = 443 [transports.http3] enabled = true -port = 443 +port = 4443 path = "/tunnel" [transports.tls_tcp] diff --git a/examples/server.minimal.toml b/examples/server.minimal.toml index f7665bf..252bb26 100644 --- a/examples/server.minimal.toml +++ b/examples/server.minimal.toml @@ -1,13 +1,17 @@ # Minimal server. Replace the backend and client secret path before use. -# The listener address/port is an example; open only the transports you need. +# This is a host address; the transport section below selects the port. [server] -listen = "0.0.0.0:51830" +listen = "0.0.0.0" [backend] target = "127.0.0.1:51820" -# Milestone 1 direct-UDP deployment. Other transports are planned or optional. +# Direct UDP only. Other transports can be enabled explicitly when needed. +[transports.direct_udp] +enabled = true +port = 51830 + [transports.quic] enabled = false diff --git a/packaging/systemd/wgcloak-client.service b/packaging/systemd/wgcloak-client.service index ccfd4a7..35972df 100644 --- a/packaging/systemd/wgcloak-client.service +++ b/packaging/systemd/wgcloak-client.service @@ -15,7 +15,7 @@ UMask=0077 # Keep the control socket in a private runtime directory. RuntimeDirectory=wgcloak-client -RuntimeDirectoryMode=0750 +RuntimeDirectoryMode=0700 Environment=XDG_RUNTIME_DIR=/run/wgcloak-client # Process and filesystem hardening. Review paths if the deployment changes. diff --git a/packaging/systemd/wgcloak-server.service b/packaging/systemd/wgcloak-server.service index 128f9fd..7d53e4c 100644 --- a/packaging/systemd/wgcloak-server.service +++ b/packaging/systemd/wgcloak-server.service @@ -15,7 +15,7 @@ UMask=0077 # Keep the control socket in a private runtime directory. RuntimeDirectory=wgcloak-server -RuntimeDirectoryMode=0750 +RuntimeDirectoryMode=0700 Environment=XDG_RUNTIME_DIR=/run/wgcloak-server # Process and filesystem hardening. Review paths if the deployment changes. diff --git a/src/cli.rs b/src/cli.rs index 22f37a8..52facb7 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -50,7 +50,7 @@ pub enum Command { #[arg(short, long)] socket: Option, }, - /// Request configuration reload. + /// Request configuration reload (currently unsupported). Reload { #[arg(short, long)] socket: Option, @@ -71,12 +71,12 @@ mod tests { fn parses_runtime_commands_and_config_paths() { let cli = Cli::try_parse_from(["wgcloak", "client", "--config", "client.toml"]).unwrap(); assert!( - matches!(cli.command, Command::Client { config } if config == PathBuf::from("client.toml")) + matches!(cli.command, Command::Client { config } if config == std::path::Path::new("client.toml")) ); let cli = Cli::try_parse_from(["wgcloak", "check-config", "-c", "server.toml"]).unwrap(); assert!( - matches!(cli.command, Command::CheckConfig { config } if config == PathBuf::from("server.toml")) + matches!(cli.command, Command::CheckConfig { config } if config == std::path::Path::new("server.toml")) ); } diff --git a/src/client/mod.rs b/src/client/mod.rs index bf9e4e8..cb3f67b 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use anyhow::Result; use bytes::Bytes; @@ -8,39 +8,148 @@ use tracing::{debug, info, warn}; use crate::{ config::{ClientConfig, TransportMode}, control::{ControlServer, ControlState}, + stats::Stats, transport::{ - quic::QuicClientConnector, DirectUdpConnector, TlsTcpConnector, TransportConnector, - TransportError, TransportSession, + quic::QuicClientConnector, DirectUdpConnector, Http3ClientConnector, TlsTcpConnector, + TransportConnector, TransportError, TransportSession, }, }; pub mod supervisor; +use supervisor::ReconnectBackoff; pub async fn run(config: ClientConfig) -> Result<()> { + run_with_http3_connector_and_control_path(config, crate::control::default_socket_path(), None) + .await +} + +/// Client entrypoint with an isolated local control socket and optional +/// caller-supplied HTTP/3 connector. The connector override is useful for +/// managed trust stores and is also used by loopback runtime tests. +pub async fn run_with_http3_connector_and_control_path( + config: ClientConfig, + control_path: impl Into, + http3_connector: Option>, +) -> Result<()> { + run_with_connector_override_and_control_path( + config, + control_path, + http3_connector.map(|connector| (TransportMode::Http3, connector)), + ) + .await +} + +/// Run the real client loop with a caller-supplied connector for one concrete +/// transport. Production normally uses the built-in connector; this hook is +/// useful for embedders with explicit trust stores and runtime integration +/// tests that verify normal certificate and hostname checks. +pub async fn run_with_transport_connector_and_control_path( + config: ClientConfig, + control_path: impl Into, + transport: TransportMode, + connector: Arc, +) -> Result<()> { + if transport == TransportMode::Auto { + anyhow::bail!("runtime connector override requires a concrete transport"); + } + run_with_connector_override_and_control_path(config, control_path, Some((transport, connector))) + .await +} + +async fn run_with_connector_override_and_control_path( + config: ClientConfig, + control_path: impl Into, + connector_override: Option<(TransportMode, Arc)>, +) -> Result<()> { + config.validate()?; let local = UdpSocket::bind(config.local.listen).await?; info!(local_addr=%local.local_addr()?, "client local UDP listener started"); + let control_state = ControlState::default(); - let control = - ControlServer::bind(crate::control::default_socket_path(), control_state.clone()).await?; - tokio::spawn(async move { - let _ = control.run().await; - }); + control_state.enable_reconnect(); + let control = ControlServer::bind(control_path, control_state.clone()).await?; + let mut control_task = tokio::spawn(control.run()); + let stats = Arc::new(Stats::default()); + control_state + .set_status(format!("starting; server={}", config.client.server)) + .await; + + let result = run_loop( + &config, + &local, + &control_state, + &stats, + connector_override + .as_ref() + .map(|(mode, connector)| (*mode, connector.as_ref())), + ) + .await; + control_state.set_status("stopping").await; + control_state.shutdown.cancel(); + // Control handlers are local but untrusted; bound the join in case a + // client stalls while writing a request. + if time::timeout(Duration::from_secs(2), &mut control_task) + .await + .is_err() + { + control_task.abort(); + } + control_state.set_status("stopped").await; + result +} + +async fn run_loop( + config: &ClientConfig, + local: &UdpSocket, + control: &ControlState, + stats: &Stats, + connector_override: Option<(TransportMode, &dyn TransportConnector)>, +) -> Result<()> { let mut latest_peer = None; - let mut backoff = Duration::from_millis(250); + let mut backoff = ReconnectBackoff::default(); let signal = shutdown_signal(); tokio::pin!(signal); + let mut refresh = time::interval(Duration::from_secs(1)); + loop { + control + .set_status(format!("probing; server={}", config.client.server)) + .await; let connected = tokio::select! { _ = &mut signal => return Ok(()), - _ = control_state.shutdown.cancelled() => return Ok(()), - result = connect_for_mode(&config) => result, + _ = control.shutdown.cancelled() => return Ok(()), + _ = control.reconnect.notified() => { + backoff.reset(); + continue; + } + _ = refresh.tick() => { + publish_client_stats(control, stats, config, "probing", None, None, None).await; + continue; + } + result = connect_for_mode(config, connector_override) => result, }; let (session, mode) = match connected { Ok(value) => { - backoff = Duration::from_millis(250); + backoff.reset(); value } Err(TransportError::AuthenticationRejected(reason)) => { + stats + .authentication_failures + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + stats + .datagrams_dropped + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + publish_client_stats( + control, + stats, + config, + "authentication_rejected", + None, + None, + None, + ) + .await; warn!(?reason, "server rejected client authentication"); return Err(anyhow::anyhow!(TransportError::AuthenticationRejected( reason @@ -48,87 +157,335 @@ pub async fn run(config: ClientConfig) -> Result<()> { } Err(error @ TransportError::Unsupported(_)) => return Err(error.into()), Err(error) => { - warn!(error=%error, retry_in_ms=backoff.as_millis(), "configured transports unavailable"); - tokio::select! { _ = &mut signal => return Ok(()), _ = control_state.shutdown.cancelled() => return Ok(()), _ = time::sleep(backoff) => {} } - backoff = (backoff * 2).min(Duration::from_secs(30)); + let delay = backoff.next_delay(); + warn!(error=%error, retry_in_ms=delay.as_millis(), "configured transports unavailable"); + control + .set_status(format!( + "reconnecting; server={}; error={error}; retry_ms={}", + config.client.server, + delay.as_millis() + )) + .await; + publish_client_stats(control, stats, config, "reconnecting", None, None, None) + .await; + tokio::select! { + _ = &mut signal => return Ok(()), + _ = control.shutdown.cancelled() => return Ok(()), + _ = control.reconnect.notified() => backoff.reset(), + _ = time::sleep(delay) => {}, + } continue; } }; info!(?mode, "client transport connected"); - let outcome = - run_connected(&local, session.as_ref(), &mut latest_peer, &control_state).await; + let connected_at = tokio::time::Instant::now(); + control + .set_status(format!( + "connected; server={}; transport={}", + config.client.server, + transport_name(mode), + )) + .await; + publish_client_stats( + control, + stats, + config, + "connected", + Some(mode), + Some(Duration::ZERO), + Some(session.as_ref()), + ) + .await; + + let outcome = run_connected( + local, + session.as_ref(), + &mut latest_peer, + control, + stats, + config, + mode, + connected_at, + ) + .await; let _ = session.close().await; - if outcome { - return Ok(()); + match outcome { + ConnectedOutcome::Shutdown => return Ok(()), + ConnectedOutcome::Reconnect => { + stats + .reconnections + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let delay = backoff.next_delay(); + control + .set_status(format!( + "reconnecting; server={}; previous_transport={}; retry_ms={}", + config.client.server, + transport_name(mode), + delay.as_millis() + )) + .await; + warn!(?mode, "transport session lost; reconnecting"); + tokio::select! { + _ = &mut signal => return Ok(()), + _ = control.shutdown.cancelled() => return Ok(()), + _ = control.reconnect.notified() => backoff.reset(), + _ = time::sleep(delay) => {}, + } + } } - warn!(?mode, "transport session lost; reconnecting"); } } async fn connect_for_mode( config: &ClientConfig, + connector_override: Option<(TransportMode, &dyn TransportConnector)>, ) -> Result<(Box, TransportMode), TransportError> { let modes = match config.transport.mode { TransportMode::Auto => config.transport.auto.order.clone(), mode => vec![mode], }; + let deadline = Duration::from_millis(config.transport.auto.probe_timeout_ms.max(1)); + let retries = config.transport.auto.retries.max(1); let mut last = TransportError::Unsupported("no transport configured".into()); for mode in modes { - let result = match mode { - TransportMode::DirectUdp => DirectUdpConnector::new(config.clone()).connect().await, - TransportMode::Quic => QuicClientConnector::new(config.clone()).connect().await, - TransportMode::TlsTcp => TlsTcpConnector::new(config.clone()).connect().await, - TransportMode::Http3 => Err(TransportError::Unsupported( - "HTTP/3 CONNECT-UDP adapter requires h3-quinn datagram integration".into(), - )), - TransportMode::Auto => Err(TransportError::Unsupported( - "nested auto transport mode".into(), - )), - }; - match result { - Ok(session) => return Ok((session, mode)), - Err(TransportError::AuthenticationRejected(reason)) => { - return Err(TransportError::AuthenticationRejected(reason)) - } - Err(error) => { - debug!(?mode, error=%error, "transport probe failed"); - last = error; + for _attempt in 0..retries { + let result = + time::timeout(deadline, connect_one(config, mode, connector_override)).await; + let result = match result { + Ok(value) => value, + Err(_) => Err(TransportError::HandshakeTimeout), + }; + match result { + Ok(session) => return Ok((session, mode)), + Err(TransportError::AuthenticationRejected(reason)) => { + return Err(TransportError::AuthenticationRejected(reason)); + } + Err(error) => { + debug!(?mode, error=%error, "transport probe failed"); + last = error; + } } } } Err(last) } +async fn connect_one( + config: &ClientConfig, + mode: TransportMode, + connector_override: Option<(TransportMode, &dyn TransportConnector)>, +) -> Result, TransportError> { + if let Some((override_mode, connector)) = connector_override { + if override_mode == mode { + return connector.connect().await; + } + } + match mode { + TransportMode::DirectUdp => DirectUdpConnector::new(config.clone()).connect().await, + TransportMode::Quic => QuicClientConnector::new(config.clone()).connect().await, + TransportMode::TlsTcp => TlsTcpConnector::new(config.clone()).connect().await, + TransportMode::Http3 => Http3ClientConnector::new(config.clone()).connect().await, + TransportMode::Auto => Err(TransportError::Unsupported( + "nested auto transport mode".into(), + )), + } +} + +#[derive(Clone, Copy)] +enum ConnectedOutcome { + Shutdown, + Reconnect, +} + +#[allow(clippy::too_many_arguments)] async fn run_connected( local: &UdpSocket, session: &dyn TransportSession, latest_peer: &mut Option, control: &ControlState, -) -> bool { + stats: &Stats, + config: &ClientConfig, + mode: TransportMode, + connected_at: tokio::time::Instant, +) -> ConnectedOutcome { let mut local_buf = vec![0u8; session.max_datagram_size().saturating_add(1)]; let signal = shutdown_signal(); tokio::pin!(signal); + let mut refresh = time::interval(Duration::from_secs(1)); + let mut keepalive = session + .authenticated_keepalive_interval() + .filter(|interval| !interval.is_zero()) + .map(|interval| time::interval_at(time::Instant::now() + interval, interval)); loop { tokio::select! { - _ = &mut signal => return true, - _ = control.shutdown.cancelled() => return true, - _ = control.reconnect.notified() => return false, - _ = control.reload.notified() => return false, + _ = &mut signal => return ConnectedOutcome::Shutdown, + _ = control.shutdown.cancelled() => return ConnectedOutcome::Shutdown, + _ = control.reconnect.notified() => return ConnectedOutcome::Reconnect, + _ = refresh.tick() => { + let duration = connected_at.elapsed(); + publish_client_stats( + control, + stats, + config, + "connected", + Some(mode), + Some(duration), + Some(session), + ).await; + } + _ = async { + match keepalive.as_mut() { + Some(interval) => interval.tick().await, + None => std::future::pending().await, + } + } => { + // Once enqueued, the TLS/TCP writer actor completes the full + // control frame even if this caller's wait is cancelled. + match time::timeout(Duration::from_secs(2), session.send_keepalive()).await { + Ok(Ok(())) => {} + Ok(Err(error)) => { + debug!(%error, "authenticated transport keepalive failed"); + return ConnectedOutcome::Reconnect; + } + Err(_) => { + debug!("authenticated transport keepalive timed out"); + return ConnectedOutcome::Reconnect; + } + } + } received = local.recv_from(&mut local_buf) => { - let (len, peer) = match received { Ok(v) => v, Err(_) => return false }; - if latest_peer.map(|old| old != peer).unwrap_or(false) { debug!(old=?latest_peer, new_addr=%peer, "local UDP peer changed; using newest peer"); } + let (len, peer) = match received { + Ok(v) => v, + Err(error) => { + stats.datagrams_dropped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + warn!(%error, "local UDP receive failed"); + return ConnectedOutcome::Reconnect; + } + }; + if latest_peer.map(|old| old != peer).unwrap_or(false) { + debug!(old=?latest_peer, new_addr=%peer, "local UDP peer changed; using newest peer"); + } *latest_peer = Some(peer); - if len > session.max_datagram_size() { warn!(len, maximum=session.max_datagram_size(), "dropping oversized local datagram"); continue; } - if session.send_datagram(Bytes::copy_from_slice(&local_buf[..len])).await.is_err() { return false; } + if len > session.max_datagram_size() { + stats.oversized_datagrams.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + stats.datagrams_dropped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + warn!(len, maximum=session.max_datagram_size(), "dropping oversized local datagram"); + continue; + } + // Do not cancel a partially written framed datagram merely + // because a control request arrives. Let it finish, but cap + // the transport operation so shutdown remains bounded. + let sent = time::timeout( + Duration::from_secs(2), + session.send_datagram(Bytes::copy_from_slice(&local_buf[..len])), + ) + .await; + match sent { + Ok(Ok(())) => {} + Ok(Err(error)) => { + stats.datagrams_dropped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + debug!(%error, "transport send failed"); + return ConnectedOutcome::Reconnect; + } + Err(_) => { + stats.datagrams_dropped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + debug!("transport send timed out"); + return ConnectedOutcome::Reconnect; + } + } + stats.add_sent(len); } received = session.recv_datagram() => { - let payload = match received { Ok(v) => v, Err(_) => return false }; - if let Some(peer) = *latest_peer { if local.send_to(&payload, peer).await.is_err() { return false; } } + let payload = match received { + Ok(v) => v, + Err(error) => { + stats.datagrams_dropped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + debug!(%error, "transport receive failed"); + return ConnectedOutcome::Reconnect; + } + }; + if payload.len() > session.max_datagram_size() { + stats.oversized_datagrams.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + stats.datagrams_dropped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + continue; + } + if let Some(peer) = *latest_peer { + match local.send_to(&payload, peer).await { + Ok(_) => stats.add_received(payload.len()), + Err(error) => { + stats.datagrams_dropped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + debug!(%error, "local UDP delivery failed"); + } + } + } else { + stats.datagrams_dropped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } } } } } +async fn publish_client_stats( + control: &ControlState, + stats: &Stats, + config: &ClientConfig, + state: &str, + mode: Option, + duration: Option, + session: Option<&dyn TransportSession>, +) { + let s = stats.snapshot(); + let transport = mode + .map(|v| format!("\"{}\"", transport_name(v))) + .unwrap_or_else(|| "null".into()); + let duration_ms = duration.map(|v| v.as_millis()).unwrap_or(0); + let remote_addr = session + .and_then(TransportSession::remote_addr) + .map(|value| format!("\"{}\"", json_escape(&value.to_string()))) + .unwrap_or_else(|| "null".into()); + let rtt_ms = session + .and_then(TransportSession::rtt) + .map(|value| value.as_millis().to_string()) + .unwrap_or_else(|| "null".into()); + let keepalive_interval_secs = session + .and_then(TransportSession::authenticated_keepalive_interval) + .map(|value| value.as_secs().to_string()) + .unwrap_or_else(|| "null".into()); + control + .set_stats_json(format!( + "{{\"state\":\"{state}\",\"server\":\"{}\",\"transport\":{transport},\"connection_duration_ms\":{duration_ms},\"remote_address\":{remote_addr},\"rtt_ms\":{rtt_ms},\"authenticated_keepalive_interval_secs\":{keepalive_interval_secs},\"max_datagram_size\":{},\"bytes_sent\":{},\"bytes_received\":{},\"datagrams_sent\":{},\"datagrams_received\":{},\"datagrams_dropped\":{},\"oversized_datagrams\":{},\"authentication_failures\":{},\"reconnections\":{}}}", + json_escape(&config.client.server), + config.network.max_datagram_size, + s.bytes_sent, + s.bytes_received, + s.datagrams_sent, + s.datagrams_received, + s.datagrams_dropped, + s.oversized_datagrams, + s.authentication_failures, + s.reconnections, + )) + .await; +} + +fn transport_name(mode: TransportMode) -> &'static str { + match mode { + TransportMode::Auto => "auto", + TransportMode::DirectUdp => "direct_udp", + TransportMode::Quic => "quic", + TransportMode::Http3 => "http3", + TransportMode::TlsTcp => "tls_tcp", + } +} + +fn json_escape(value: &str) -> String { + value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n") + .replace('\r', "\\r") +} + async fn shutdown_signal() { #[cfg(unix)] { diff --git a/src/client/supervisor.rs b/src/client/supervisor.rs index ef035df..8ccad64 100644 --- a/src/client/supervisor.rs +++ b/src/client/supervisor.rs @@ -70,7 +70,32 @@ pub struct TransportSupervisor

{ status: Arc>, reconnect: Arc, shutdown: CancellationToken, - backoff_step: u32, + backoff: ReconnectBackoff, +} + +/// Exponential reconnect delay with bounded downward jitter. The first retry +/// is 250 ms; later retries grow to at most 30 seconds. A successful +/// connection resets the sequence. +#[derive(Clone, Debug, Default)] +pub struct ReconnectBackoff { + step: u32, +} +impl ReconnectBackoff { + pub fn next_delay(&mut self) -> Duration { + let n = self.step.min(17); + let base = 250u64.saturating_mul(1u64 << n).min(30_000); + let jitter = if base <= 250 { + 0 + } else { + rand::random::() % (base / 4).max(1) + }; + self.step = self.step.saturating_add(1); + Duration::from_millis(base.saturating_sub(jitter).clamp(250, 30_000)) + } + + pub fn reset(&mut self) { + self.step = 0; + } } impl TransportSupervisor

{ pub fn new(config: ClientConfig, probe: P) -> Self { @@ -80,7 +105,7 @@ impl TransportSupervisor

{ status: Arc::new(Mutex::new(SupervisorStatus::default())), reconnect: Arc::new(Notify::new()), shutdown: CancellationToken::new(), - backoff_step: 0, + backoff: ReconnectBackoff::default(), } } pub fn shutdown_token(&self) -> CancellationToken { @@ -138,18 +163,10 @@ impl TransportSupervisor

{ Err(error) } pub fn backoff_delay(&mut self) -> Duration { - let n = self.backoff_step.min(17); - let base = 250u64.saturating_mul(1u64 << n).min(30_000); - let jitter = if base <= 250 { - 0 - } else { - rand::random::() % (base / 4).max(1) - }; - self.backoff_step = self.backoff_step.saturating_add(1); - Duration::from_millis(base.saturating_sub(jitter).clamp(250, 30_000)) + self.backoff.next_delay() } pub fn reset_backoff(&mut self) { - self.backoff_step = 0; + self.backoff.reset(); } pub async fn run(mut self) -> Result<(), TransportError> { loop { diff --git a/src/config.rs b/src/config.rs index 96a9de0..5b62242 100644 --- a/src/config.rs +++ b/src/config.rs @@ -13,6 +13,11 @@ use tracing::{info, warn}; const DEFAULT_MAX_DATAGRAM_SIZE: u16 = 1200; const MIN_SECRET_BYTES: usize = 32; +const MAX_DATAGRAM_SIZE: u16 = u16::MAX - 32; +const MAX_KEEPALIVE_SECS: u64 = 86_400; +const MAX_IDLE_TIMEOUT_SECS: u64 = 86_400; +const MAX_PROBE_TIMEOUT_MS: u64 = 300_000; +const MAX_AUTO_RETRIES: u8 = 20; #[derive(Debug, Error)] pub enum ConfigError { @@ -120,14 +125,11 @@ pub struct ClientConfig { impl ClientConfig { pub fn validate(&self) -> Result<(), ConfigError> { - if self.client.server.trim().is_empty() { - return Err(ConfigError::Invalid( - "client.server must not be empty".into(), - )); - } + validate_host(&self.client.server, "client.server")?; if self.client.id.trim().is_empty() { return Err(ConfigError::Invalid("client.id must not be empty".into())); } + validate_port(self.local.listen.port(), "local.listen")?; validate_secret_source(&self.client.secret, "client")?; validate_max_datagram(self.network.max_datagram_size)?; self.transport.validate() @@ -200,11 +202,61 @@ impl ClientTransportConfig { "transport.auto.probe_timeout_ms must be > 0".into(), )); } + if self.auto.probe_timeout_ms > MAX_PROBE_TIMEOUT_MS { + return Err(ConfigError::Invalid(format!( + "transport.auto.probe_timeout_ms must be <= {MAX_PROBE_TIMEOUT_MS}" + ))); + } if self.auto.retries == 0 { return Err(ConfigError::Invalid( "transport.auto.retries must be > 0".into(), )); } + if self.auto.retries > MAX_AUTO_RETRIES { + return Err(ConfigError::Invalid(format!( + "transport.auto.retries must be <= {MAX_AUTO_RETRIES}" + ))); + } + for (index, mode) in self.auto.order.iter().enumerate() { + if *mode == TransportMode::Auto { + return Err(ConfigError::Invalid( + "transport.auto.order must contain concrete transports, not 'auto'".into(), + )); + } + if self.auto.order[..index].contains(mode) { + return Err(ConfigError::Invalid(format!( + "transport.auto.order contains duplicate transport {mode:?}" + ))); + } + } + + // Resolve the effective attempt list according to mode: an explicit + // selection is a singleton and never falls back through auto.order. + let active_modes = if self.mode == TransportMode::Auto { + self.auto.order.as_slice() + } else { + std::slice::from_ref(&self.mode) + }; + if active_modes.is_empty() || active_modes.contains(&TransportMode::Auto) { + return Err(ConfigError::Invalid( + "transport.mode must resolve to one or more concrete transports".into(), + )); + } + + validate_port(self.direct_udp.port, "transport.direct_udp.port")?; + validate_port(self.quic.port, "transport.quic.port")?; + validate_port(self.http3.port, "transport.http3.port")?; + validate_port(self.tls_tcp.port, "transport.tls_tcp.port")?; + validate_keepalive( + self.direct_udp.keepalive_secs, + "transport.direct_udp.keepalive_secs", + )?; + validate_keepalive(self.quic.keepalive_secs, "transport.quic.keepalive_secs")?; + validate_keepalive( + self.tls_tcp.keepalive_secs, + "transport.tls_tcp.keepalive_secs", + )?; + validate_http3_path(&self.http3.path, "transport.http3.path")?; Ok(()) } } @@ -292,11 +344,15 @@ impl Default for ClientHttp3Config { pub struct ClientTlsTcpConfig { #[serde(default = "default_tls_port")] pub port: u16, + /// Seconds between authenticated TLS/TCP control keepalives. Zero disables them. + #[serde(default = "default_keepalive")] + pub keepalive_secs: u64, } impl Default for ClientTlsTcpConfig { fn default() -> Self { Self { port: default_tls_port(), + keepalive_secs: default_keepalive(), } } } @@ -333,31 +389,74 @@ pub struct ServerConfig { impl ServerConfig { pub fn validate(&self) -> Result<(), ConfigError> { - if self.server.listen.trim().is_empty() { - return Err(ConfigError::Invalid( - "server.listen must not be empty".into(), - )); - } - self.backend - .target - .parse::() - .map_err(|_| ConfigError::Invalid("backend.target must be a socket address".into()))?; + validate_host(&self.server.listen, "server.listen")?; + let backend = + self.backend.target.parse::().map_err(|_| { + ConfigError::Invalid("backend.target must be a socket address".into()) + })?; + validate_port(backend.port(), "backend.target")?; if self.auth.clients.is_empty() { return Err(ConfigError::Invalid( "auth.clients must contain at least one client".into(), )); } - if self.session.idle_timeout_secs == 0 { - return Err(ConfigError::Invalid( - "session.idle_timeout_secs must be > 0".into(), - )); + if self.session.idle_timeout_secs == 0 + || self.session.idle_timeout_secs > MAX_IDLE_TIMEOUT_SECS + { + return Err(ConfigError::Invalid(format!( + "session.idle_timeout_secs must be between 1 and {MAX_IDLE_TIMEOUT_SECS}" + ))); } if self.limits.max_sessions == 0 || self.limits.max_sessions_per_ip == 0 || self.limits.max_pending_handshakes == 0 + || self.limits.max_auth_attempts_per_ip == 0 + || self.limits.auth_attempt_window_secs == 0 + || self.limits.handshake_timeout_ms == 0 { - return Err(ConfigError::Invalid("session limits must be > 0".into())); + return Err(ConfigError::Invalid( + "session and authentication limits must be > 0".into(), + )); } + if self.limits.max_sessions > 1_000_000 + || self.limits.max_sessions_per_ip > 1_000_000 + || self.limits.max_pending_handshakes > 100_000 + || self.limits.max_auth_attempts_per_ip > 100_000 + || self.limits.auth_attempt_window_secs > 86_400 + || self.limits.handshake_timeout_ms > 300_000 + { + return Err(ConfigError::Invalid( + "session and authentication limits exceed supported bounds".into(), + )); + } + if self.limits.max_sessions_per_ip > self.limits.max_sessions { + return Err(ConfigError::Invalid( + "limits.max_sessions_per_ip must be <= limits.max_sessions".into(), + )); + } + + validate_port( + self.transports.direct_udp.port, + "transports.direct_udp.port", + )?; + validate_port(self.transports.quic.port, "transports.quic.port")?; + validate_port(self.transports.http3.port, "transports.http3.port")?; + validate_port(self.transports.tls_tcp.port, "transports.tls_tcp.port")?; + validate_http3_path(&self.transports.http3.path, "transports.http3.path")?; + + let enabled_transports = [ + self.transports.direct_udp.enabled, + self.transports.quic.enabled, + self.transports.http3.enabled, + self.transports.tls_tcp.enabled, + ]; + if !enabled_transports.into_iter().any(|enabled| enabled) { + return Err(ConfigError::Invalid( + "at least one server transport must be enabled".into(), + )); + } + validate_udp_listener_ports(self)?; + let mut client_ids = HashSet::with_capacity(self.auth.clients.len()); for client in &self.auth.clients { if client.id.trim().is_empty() { @@ -485,6 +584,19 @@ impl TlsConfig { "tls.certificate and tls.private_key must be provided together".into(), )); } + if self + .certificate + .as_ref() + .is_some_and(|path| path.as_os_str().is_empty()) + || self + .private_key + .as_ref() + .is_some_and(|path| path.as_os_str().is_empty()) + { + return Err(ConfigError::Invalid( + "tls.certificate and tls.private_key paths must not be empty".into(), + )); + } Ok(()) } } @@ -510,6 +622,10 @@ pub struct LimitsConfig { pub max_sessions_per_ip: usize, #[serde(default = "default_max_pending_handshakes")] pub max_pending_handshakes: usize, + #[serde(default = "default_max_auth_attempts_per_ip")] + pub max_auth_attempts_per_ip: usize, + #[serde(default = "default_auth_attempt_window_secs")] + pub auth_attempt_window_secs: u64, #[serde(default = "default_handshake_timeout_ms")] pub handshake_timeout_ms: u64, } @@ -519,6 +635,8 @@ impl Default for LimitsConfig { max_sessions: default_max_sessions(), max_sessions_per_ip: default_max_sessions_per_ip(), max_pending_handshakes: default_max_pending_handshakes(), + max_auth_attempts_per_ip: default_max_auth_attempts_per_ip(), + auth_attempt_window_secs: default_auth_attempt_window_secs(), handshake_timeout_ms: default_handshake_timeout_ms(), } } @@ -728,17 +846,138 @@ fn default_max_sessions_per_ip() -> usize { fn default_max_pending_handshakes() -> usize { 256 } +fn default_max_auth_attempts_per_ip() -> usize { + 20 +} +fn default_auth_attempt_window_secs() -> u64 { + 60 +} fn default_handshake_timeout_ms() -> u64 { 3000 } fn default_log_level() -> String { "info".to_owned() } + +fn validate_host(value: &str, label: &str) -> Result<(), ConfigError> { + let invalid = || { + ConfigError::Invalid(format!( + "{label} must be a hostname or IP address without a port" + )) + }; + if value.is_empty() || value != value.trim() { + return Err(invalid()); + } + if value.parse::().is_ok() { + return Ok(()); + } + if value.starts_with('[') && value.ends_with(']') { + let ipv6 = &value[1..value.len() - 1]; + if ipv6.parse::().is_ok() { + return Ok(()); + } + } + if !value.is_ascii() || value.contains([':', '/', '[', ']', '@']) { + return Err(invalid()); + } + + let hostname = value.strip_suffix('.').unwrap_or(value); + if hostname.is_empty() || hostname.len() > 253 { + return Err(invalid()); + } + let labels: Vec<&str> = hostname.split('.').collect(); + if labels.iter().any(|part| { + part.is_empty() + || part.len() > 63 + || part.starts_with('-') + || part.ends_with('-') + || !part + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + }) { + return Err(invalid()); + } + // Avoid treating malformed dotted-quad literals as DNS names. + if labels.len() == 4 + && labels + .iter() + .all(|part| part.bytes().all(|b| b.is_ascii_digit())) + { + return Err(invalid()); + } + Ok(()) +} + +fn validate_port(port: u16, label: &str) -> Result<(), ConfigError> { + if port == 0 { + Err(ConfigError::Invalid(format!( + "{label} must be between 1 and 65535" + ))) + } else { + Ok(()) + } +} + +fn validate_keepalive(seconds: u64, label: &str) -> Result<(), ConfigError> { + if seconds > MAX_KEEPALIVE_SECS { + Err(ConfigError::Invalid(format!( + "{label} must be 0 (disabled) or <= {MAX_KEEPALIVE_SECS} seconds" + ))) + } else { + Ok(()) + } +} + +fn validate_http3_path(path: &str, label: &str) -> Result<(), ConfigError> { + if !path.starts_with('/') { + Err(ConfigError::Invalid(format!("{label} must begin with '/'"))) + } else { + Ok(()) + } +} + +fn validate_udp_listener_ports(config: &ServerConfig) -> Result<(), ConfigError> { + let udp_listeners = [ + ( + "direct_udp", + config.transports.direct_udp.enabled, + config.transports.direct_udp.port, + ), + ( + "quic", + config.transports.quic.enabled, + config.transports.quic.port, + ), + ( + "http3", + config.transports.http3.enabled, + config.transports.http3.port, + ), + ]; + for (index, (left_name, left_enabled, left_port)) in udp_listeners.iter().enumerate() { + if !left_enabled { + continue; + } + for (right_name, right_enabled, right_port) in udp_listeners.iter().skip(index + 1) { + if *right_enabled && left_port == right_port { + return Err(ConfigError::Invalid(format!( + "UDP listeners {left_name} and {right_name} conflict on port {left_port}; configure distinct UDP ports" + ))); + } + } + } + Ok(()) +} + fn validate_max_datagram(value: u16) -> Result<(), ConfigError> { if value == 0 { Err(ConfigError::Invalid( "network.max_datagram_size must be > 0".into(), )) + } else if value > MAX_DATAGRAM_SIZE { + Err(ConfigError::Invalid(format!( + "network.max_datagram_size must be <= {MAX_DATAGRAM_SIZE}" + ))) } else { Ok(()) } @@ -752,6 +991,40 @@ mod tests { const SECRET: &str = "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY="; + fn client_text(server: &str, extra: &str) -> String { + format!("[client]\nserver='{server}'\nid='laptop'\nsecret='{SECRET}'\n{extra}") + } + + fn server_text(listen: &str, extra: &str) -> String { + format!( + "[server]\nlisten='{listen}'\n[backend]\ntarget='127.0.0.1:51820'\n{extra}\n[[auth.clients]]\nid='laptop'\nsecret='{SECRET}'\n" + ) + } + + fn validation_error(text: &str) -> String { + Config::from_str(text) + .unwrap() + .validate() + .unwrap_err() + .to_string() + } + + fn with_test_secrets(example: &str) -> String { + example + .lines() + .map(|line| { + let trimmed = line.trim_start(); + if let Some(indent) = line.strip_suffix(trimmed) { + if trimmed.starts_with("secret_file =") { + return format!("{indent}secret = \"{SECRET}\""); + } + } + line.to_owned() + }) + .collect::>() + .join("\n") + } + #[test] fn client_defaults_and_validation_work() { let text = format!("[client]\nserver='vpn.example.com'\nid='laptop'\nsecret='{SECRET}'\n"); @@ -761,6 +1034,16 @@ mod tests { assert_eq!(client.local.listen, "127.0.0.1:51821".parse().unwrap()); assert_eq!(client.network.max_datagram_size, 1200); assert_eq!(client.transport.mode, TransportMode::Auto); + assert_eq!(client.transport.tls_tcp.keepalive_secs, 15); + } + + #[test] + fn tls_tcp_keepalive_can_be_disabled_without_breaking_old_configs() { + let text = format!( + "[client]\nserver='vpn.example.com'\nid='laptop'\nsecret='{SECRET}'\n[transport.tls_tcp]\nkeepalive_secs=0\n" + ); + let client = Config::from_str(&text).unwrap().into_client().unwrap(); + assert_eq!(client.transport.tls_tcp.keepalive_secs, 0); } #[test] @@ -809,6 +1092,226 @@ mod tests { )); } + #[test] + fn client_hostnames_and_ip_literals_are_accepted_but_ports_and_bad_syntax_are_not() { + for host in [ + "vpn.example.com", + "vpn.example.com.", + "127.0.0.1", + "::1", + "[::1]", + ] { + Config::from_str(&client_text(host, "")) + .unwrap() + .validate() + .unwrap_or_else(|error| panic!("expected {host:?} to validate: {error}")); + } + for host in [ + "", + "vpn.example.com:443", + "bad..host", + "-bad.example", + "bad host", + "256.1.1.1", + ] { + let error = validation_error(&client_text(host, "")); + assert!(error.contains("client.server"), "{host:?}: {error}"); + } + } + + #[test] + fn client_transport_order_mode_ports_paths_sizes_and_timeouts_are_validated() { + for (extra, expected) in [ + ("[transport.auto]\norder=[]\n", "must not be empty"), + ("[transport.auto]\norder=['auto']\n", "concrete transports"), + ( + "[transport.auto]\norder=['quic','quic']\n", + "duplicate transport", + ), + ("[transport.auto]\nprobe_timeout_ms=0\n", "probe_timeout_ms"), + ( + "[transport.auto]\nprobe_timeout_ms=300001\n", + "probe_timeout_ms", + ), + ("[transport.auto]\nretries=0\n", "retries"), + ("[transport.auto]\nretries=21\n", "retries"), + ("[local]\nlisten='127.0.0.1:0'\n", "local.listen"), + ( + "[transport.direct_udp]\nport=0\n", + "transport.direct_udp.port", + ), + ("[transport.quic]\nport=0\n", "transport.quic.port"), + ("[transport.http3]\nport=0\n", "transport.http3.port"), + ("[transport.tls_tcp]\nport=0\n", "transport.tls_tcp.port"), + ("[transport.http3]\npath='tunnel'\n", "transport.http3.path"), + ( + "[transport.direct_udp]\nkeepalive_secs=86401\n", + "keepalive_secs", + ), + ("[transport.quic]\nkeepalive_secs=86401\n", "keepalive_secs"), + ( + "[transport.tls_tcp]\nkeepalive_secs=86401\n", + "keepalive_secs", + ), + ("[network]\nmax_datagram_size=0\n", "max_datagram_size"), + ("[network]\nmax_datagram_size=65504\n", "max_datagram_size"), + ] { + let error = validation_error(&client_text("vpn.example.com", extra)); + assert!(error.contains(expected), "expected {expected:?}: {error}"); + } + + // A concrete mode selects exactly itself; auto.order remains validated + // but does not cause fallback when mode is explicit. + Config::from_str(&client_text( + "vpn.example.com", + "[transport]\nmode='quic'\n[transport.auto]\norder=['direct_udp']\n", + )) + .unwrap() + .validate() + .unwrap(); + Config::from_str(&client_text( + "vpn.example.com", + "[network]\nmax_datagram_size=65503\n", + )) + .unwrap() + .validate() + .expect("the direct packet payload ceiling is a valid limit"); + } + + #[test] + fn server_host_backend_enabled_transports_ports_paths_and_tls_are_validated() { + for host in ["vpn.example.com", "127.0.0.1", "::1", "[::1]"] { + Config::from_str(&server_text(host, "")) + .unwrap() + .validate() + .unwrap_or_else(|error| panic!("expected {host:?} to validate: {error}")); + } + for host in [ + "", + "127.0.0.1:443", + "[::1]:443", + "[not-ip]", + "bad..host", + "bad host", + ] { + let error = validation_error(&server_text(host, "")); + assert!(error.contains("server.listen"), "{host:?}: {error}"); + } + + for target in ["not-an-address", "127.0.0.1:0"] { + let text = server_text("127.0.0.1", "") + .replace("target='127.0.0.1:51820'", &format!("target='{target}'")); + let error = validation_error(&text); + assert!(error.contains("backend.target"), "{target:?}: {error}"); + } + + for (extra, expected) in [ + ( + "[transports.direct_udp]\nenabled=false\n[transports.quic]\nenabled=false\n[transports.http3]\nenabled=false\n[transports.tls_tcp]\nenabled=false\n", + "at least one server transport", + ), + ( + "[transports.direct_udp]\nport=0\n", + "transports.direct_udp.port", + ), + ("[transports.quic]\nport=0\n", "transports.quic.port"), + ("[transports.http3]\nport=0\n", "transports.http3.port"), + ("[transports.tls_tcp]\nport=0\n", "transports.tls_tcp.port"), + ( + "[transports.http3]\npath='tunnel'\n", + "transports.http3.path", + ), + ( + "[transports.quic]\nenabled=true\n", + "tls.certificate and tls.private_key are required", + ), + ( + "[transports.http3]\nenabled=true\n", + "tls.certificate and tls.private_key are required", + ), + ( + "[transports.tls_tcp]\nenabled=true\n", + "tls.certificate and tls.private_key are required", + ), + ( + "[transports.direct_udp]\nport=51830\n[transports.quic]\nenabled=true\nport=51830\n", + "UDP listeners direct_udp and quic", + ), + ( + "[transports.quic]\nenabled=true\nport=443\n[transports.http3]\nenabled=true\nport=443\n", + "UDP listeners quic and http3", + ), + ( + "[transports.direct_udp]\nport=51830\n[transports.http3]\nenabled=true\nport=51830\n", + "UDP listeners direct_udp and http3", + ), + ( + "[limits]\nmax_sessions=10\nmax_sessions_per_ip=11\n", + "max_sessions_per_ip must be <= limits.max_sessions", + ), + ("[limits]\nmax_sessions=0\n", "must be > 0"), + ( + "[limits]\nmax_sessions_per_ip=1000001\n", + "supported bounds", + ), + ("[limits]\nmax_pending_handshakes=100001\n", "supported bounds"), + ("[limits]\nmax_auth_attempts_per_ip=0\n", "must be > 0"), + ( + "[limits]\nmax_auth_attempts_per_ip=100001\n", + "supported bounds", + ), + ( + "[limits]\nauth_attempt_window_secs=86401\n", + "supported bounds", + ), + ( + "[limits]\nhandshake_timeout_ms=300001\n", + "supported bounds", + ), + ("[session]\nidle_timeout_secs=0\n", "idle_timeout_secs"), + ("[session]\nidle_timeout_secs=86401\n", "idle_timeout_secs"), + ( + "[transports.quic]\nenabled=true\n[tls]\ncertificate='/tmp/cert.pem'\n", + "must be provided together", + ), + ] { + let error = validation_error(&server_text("127.0.0.1", extra)); + assert!(error.contains(expected), "expected {expected:?}: {error}"); + } + + Config::from_str(&server_text( + "127.0.0.1", + "[transports.quic]\nenabled=true\nport=443\n[transports.tls_tcp]\nenabled=true\nport=443\n[tls]\ncertificate='/tmp/cert.pem'\nprivate_key='/tmp/key.pem'\n", + )) + .unwrap() + .validate() + .expect("TCP and UDP may share a numeric port"); + + Config::from_str(&server_text( + "127.0.0.1", + "[limits]\nmax_sessions=10\nmax_sessions_per_ip=10\n", + )) + .unwrap() + .validate() + .expect("per-IP limit may equal total session limit"); + } + + #[test] + fn shipped_examples_parse_and_satisfy_configuration_contract() { + for example in [ + include_str!("../examples/client.minimal.toml"), + include_str!("../examples/client.advanced.toml"), + include_str!("../examples/server.minimal.toml"), + include_str!("../examples/server.advanced.toml"), + ] { + let normalized = with_test_secrets(example); + Config::from_str(&normalized) + .unwrap() + .validate() + .unwrap_or_else(|error| panic!("example did not validate: {error}\n{example}")); + } + } + #[test] fn secret_file_must_be_private() { let file = NamedTempFile::new().unwrap(); diff --git a/src/control.rs b/src/control.rs index d7fcf05..49b538a 100644 --- a/src/control.rs +++ b/src/control.rs @@ -1,11 +1,17 @@ //! Bounded newline-delimited JSON control socket for local administration. +use crate::stats::StatsSnapshot; #[cfg(unix)] -use std::os::unix::fs::FileTypeExt; +use std::os::unix::fs::{FileTypeExt, MetadataExt}; use std::{ + collections::BTreeMap, io, path::{Path, PathBuf}, - sync::Arc, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, }; use tokio::{ io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}, @@ -19,18 +25,27 @@ use tokio_util::sync::CancellationToken; pub struct ControlState { status: Arc>, stats: Arc>, + transport_stats: Arc>>, pub reconnect: Arc, - pub reload: Arc, pub shutdown: CancellationToken, + reconnect_supported: Arc, +} + +#[derive(Clone, Default)] +struct TransportStats { + active_sessions: usize, + counters: StatsSnapshot, + active_sessions_by_client_id: BTreeMap, } impl Default for ControlState { fn default() -> Self { Self { status: Arc::new(Mutex::new("starting".into())), stats: Arc::new(Mutex::new("{}".into())), + transport_stats: Arc::new(Mutex::new(BTreeMap::new())), reconnect: Arc::new(Notify::new()), - reload: Arc::new(Notify::new()), shutdown: CancellationToken::new(), + reconnect_supported: Arc::new(AtomicBool::new(false)), } } } @@ -41,16 +56,173 @@ impl ControlState { pub async fn set_stats_json(&self, value: impl Into) { *self.stats.lock().await = value.into(); } + + /// Publish one listener's latest counters. The control-plane view retains + /// every enabled transport and emits a deterministic aggregate snapshot. + pub async fn set_transport_stats( + &self, + transport: impl Into, + active_sessions: usize, + snapshot: StatsSnapshot, + ) { + self.update_transport_stats(transport.into(), active_sessions, snapshot, None) + .await; + } + + /// Publish listener counters together with its currently authenticated + /// sessions by configured client identity. + pub async fn set_transport_stats_by_client_id( + &self, + transport: impl Into, + active_sessions: usize, + snapshot: StatsSnapshot, + active_sessions_by_client_id: BTreeMap, + ) { + self.update_transport_stats( + transport.into(), + active_sessions, + snapshot, + Some(active_sessions_by_client_id), + ) + .await; + } + + async fn update_transport_stats( + &self, + transport: String, + active_sessions: usize, + snapshot: StatsSnapshot, + client_sessions: Option>, + ) { + let mut transports = self.transport_stats.lock().await; + let entry = transports.entry(transport).or_default(); + entry.active_sessions = active_sessions; + entry.counters = snapshot; + if let Some(client_sessions) = client_sessions { + entry.active_sessions_by_client_id = client_sessions; + } + let mut aggregate = StatsSnapshot::default(); + let mut active_total = 0usize; + let mut by_transport = Vec::with_capacity(transports.len()); + let mut per_transport = Vec::with_capacity(transports.len()); + let mut by_client_id = BTreeMap::::new(); + let mut by_transport_and_client = Vec::with_capacity(transports.len()); + for (name, stats) in transports.iter() { + active_total = active_total.saturating_add(stats.active_sessions); + aggregate = aggregate.saturating_add(stats.counters); + by_transport.push(format!("{}:{}", json_string(name), stats.active_sessions)); + per_transport.push(format!( + "{}:{{{}}}", + json_string(name), + snapshot_fields(stats.counters) + )); + let client_entries = stats + .active_sessions_by_client_id + .iter() + .map(|(client_id, count)| { + let total = by_client_id.entry(client_id.clone()).or_default(); + *total = total.saturating_add(*count); + format!("{}:{count}", json_string(client_id)) + }) + .collect::>(); + by_transport_and_client.push(format!( + "{}:{{{}}}", + json_string(name), + client_entries.join(",") + )); + } + let by_client_id = by_client_id + .iter() + .map(|(client_id, count)| format!("{}:{count}", json_string(client_id))) + .collect::>() + .join(","); + let label = if transports.len() == 1 { + transports + .keys() + .next() + .map(String::as_str) + .unwrap_or("unknown") + } else { + "multi" + }; + let json = format!( + "{{\"transport\":\"{label}\",\"active_sessions\":{active_total},\"active_sessions_by_transport\":{{{}}},\"active_sessions_by_client_id\":{{{by_client_id}}},\"active_sessions_by_transport_and_client_id\":{{{}}},\"transports\":{{{}}},{}}}", + by_transport.join(","), + by_transport_and_client.join(","), + per_transport.join(","), + snapshot_fields(aggregate), + ); + *self.stats.lock().await = json; + } + + async fn status_with_activity(&self) -> String { + let status = self.status.lock().await.clone(); + if !status.starts_with("listening;") { + return status; + } + let transports = self.transport_stats.lock().await; + if transports.is_empty() { + return status; + } + let active_total = transports + .values() + .map(|stats| stats.active_sessions) + .sum::(); + let by_transport = transports + .iter() + .map(|(name, stats)| format!("{name}:{}", stats.active_sessions)) + .collect::>() + .join(","); + format!( + "{status}; active_sessions={active_total}; active_sessions_by_transport={by_transport}" + ) + } + + pub fn enable_reconnect(&self) { + self.reconnect_supported.store(true, Ordering::Release); + } +} + +fn json_string(value: &str) -> String { + format!( + "\"{}\"", + value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n") + .replace('\r', "\\r") + ) +} + +fn snapshot_fields(snapshot: StatsSnapshot) -> String { + format!( + "\"bytes_sent\":{},\"bytes_received\":{},\"datagrams_sent\":{},\"datagrams_received\":{},\"datagrams_dropped\":{},\"oversized_datagrams\":{},\"authentication_failures\":{},\"reconnections\":{},\"expired_sessions\":{},\"handshake_attempts\":{},\"authentication_rejections\":{},\"rate_limited_attempts\":{},\"backend_errors\":{}", + snapshot.bytes_sent, + snapshot.bytes_received, + snapshot.datagrams_sent, + snapshot.datagrams_received, + snapshot.datagrams_dropped, + snapshot.oversized_datagrams, + snapshot.authentication_failures, + snapshot.reconnections, + snapshot.expired_sessions, + snapshot.handshake_attempts, + snapshot.authentication_rejections, + snapshot.rate_limited_attempts, + snapshot.backend_errors, + ) } pub struct ControlServer { listener: UnixListener, path: PathBuf, + socket_identity: (u64, u64), state: ControlState, } impl ControlServer { pub async fn bind(path: impl Into, state: ControlState) -> io::Result { let path = path.into(); + prepare_socket_parent(&path)?; if let Ok(metadata) = std::fs::symlink_metadata(&path) { if !metadata.file_type().is_socket() { return Err(io::Error::new( @@ -58,17 +230,57 @@ impl ControlServer { "control path is not a socket", )); } - std::fs::remove_file(&path)?; + match tokio::time::timeout(Duration::from_millis(250), UnixStream::connect(&path)).await + { + Ok(Ok(_)) => { + return Err(io::Error::new( + io::ErrorKind::AddrInUse, + "control socket is already active", + )); + } + Ok(Err(error)) + if matches!( + error.kind(), + io::ErrorKind::ConnectionRefused | io::ErrorKind::NotFound + ) => + { + let current = std::fs::symlink_metadata(&path)?; + if !current.file_type().is_socket() + || current.dev() != metadata.dev() + || current.ino() != metadata.ino() + { + return Err(io::Error::new( + io::ErrorKind::AddrInUse, + "control path changed while checking for a stale socket", + )); + } + std::fs::remove_file(&path)?; + } + Ok(Err(error)) => return Err(error), + Err(_) => { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "timed out checking whether the control socket is active", + )); + } + } } let listener = UnixListener::bind(&path)?; #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600))?; + if let Err(error) = + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + { + let _ = std::fs::remove_file(&path); + return Err(error); + } } + let metadata = std::fs::symlink_metadata(&path)?; Ok(Self { listener, path, + socket_identity: (metadata.dev(), metadata.ino()), state, }) } @@ -77,16 +289,84 @@ impl ControlServer { loop { tokio::select! { _ = self.state.shutdown.cancelled() => break, Some(result) = tasks.join_next() => { if let Err(error) = result { tracing::debug!(%error, "control connection ended"); } }, accepted = self.listener.accept() => { let (stream, _) = accepted?; let state = self.state.clone(); tasks.spawn(async move { let _ = handle_connection(stream, state).await; }); } } } + // A client may connect and then never finish writing a command. Do not + // let that local connection pin daemon shutdown indefinitely. + tasks.abort_all(); while tasks.join_next().await.is_some() {} Ok(()) } } impl Drop for ControlServer { fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); + if let Ok(metadata) = std::fs::symlink_metadata(&self.path) { + if metadata.file_type().is_socket() + && (metadata.dev(), metadata.ino()) == self.socket_identity + { + let _ = std::fs::remove_file(&self.path); + } + } } } +fn prepare_socket_parent(path: &Path) -> io::Result<()> { + use std::os::unix::fs::PermissionsExt; + + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let existed = match std::fs::symlink_metadata(parent) { + Ok(_) => true, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + std::fs::create_dir_all(parent)?; + false + } + Err(error) => return Err(error), + }; + let metadata = std::fs::symlink_metadata(parent)?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "control socket parent must be a real directory", + )); + } + if metadata.uid() != current_uid()? { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "control socket parent must be owned by the current user", + )); + } + if existed && metadata.mode() & 0o777 != 0o700 { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "control socket parent must have mode 0700", + )); + } + if !existed { + std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700))?; + } + let verified = std::fs::symlink_metadata(parent)?; + if !verified.is_dir() || verified.uid() != current_uid()? || verified.mode() & 0o777 != 0o700 { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "control socket parent could not be made private", + )); + } + Ok(()) +} + +#[cfg(unix)] +fn current_uid() -> io::Result { + std::fs::metadata("/proc/self") + .map(|metadata| metadata.uid()) + .or_else(|_| { + std::env::var("UID") + .ok() + .and_then(|value| value.parse().ok()) + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "cannot determine uid")) + }) +} + async fn handle_connection(stream: UnixStream, state: ControlState) -> io::Result<()> { let (read, mut write) = stream.into_split(); let mut reader = BufReader::new(read).take(8192); @@ -97,17 +377,23 @@ async fn handle_connection(stream: UnixStream, state: ControlState) -> io::Resul } else { match command_name(&line) { Some("status") => { - let value = state.status.lock().await.clone(); + let value = state.status_with_activity().await; format!("{{\"ok\":true,\"status\":\"{}\"}}\n", json_escape(&value)) } Some("stats") => format!("{{\"ok\":true,\"stats\":{}}}\n", state.stats.lock().await), Some("reconnect") => { - state.reconnect.notify_one(); - "{\"ok\":true,\"command\":\"reconnect\"}\n".into() + if state.reconnect_supported.load(Ordering::Acquire) { + state.reconnect.notify_one(); + "{\"ok\":true,\"command\":\"reconnect\"}\n".into() + } else { + "{\"ok\":false,\"error\":\"reconnect is not supported by this daemon\"}\n" + .into() + } } + // Reload is deliberately rejected until there is a file-backed + // runtime configuration source and an atomic apply path. Some("reload") => { - state.reload.notify_one(); - "{\"ok\":true,\"command\":\"reload\"}\n".into() + "{\"ok\":false,\"error\":\"configuration reload is not supported\"}\n".into() } Some("shutdown") => { state.shutdown.cancel(); @@ -154,10 +440,43 @@ fn json_escape(value: &str) -> String { } pub fn default_socket_path() -> PathBuf { - std::env::var_os("XDG_RUNTIME_DIR") + let uid = current_uid().unwrap_or(0); + if let Some(runtime_dir) = std::env::var_os("XDG_RUNTIME_DIR") + .filter(|path| !path.is_empty()) .map(PathBuf::from) - .unwrap_or_else(|| PathBuf::from("/tmp")) - .join("wgcloak.sock") + { + if is_private_runtime_dir(&runtime_dir, uid) { + return runtime_dir.join("wgcloak.sock"); + } + } + let systemd_runtime = PathBuf::from(format!("/run/user/{uid}")); + if is_private_runtime_dir(&systemd_runtime, uid) { + return systemd_runtime.join("wgcloak/wgcloak.sock"); + } + if let Some(home) = std::env::var_os("HOME").filter(|path| !path.is_empty()) { + let home = PathBuf::from(home); + let in_shared_tmp = std::fs::canonicalize(&home) + .ok() + .zip(std::fs::canonicalize(std::env::temp_dir()).ok()) + .map(|(home, temporary)| home.starts_with(temporary)) + .unwrap_or(false); + if !in_shared_tmp { + return home.join(".local/run/wgcloak/wgcloak.sock"); + } + } + systemd_runtime.join("wgcloak/wgcloak.sock") +} + +#[cfg(unix)] +fn is_private_runtime_dir(path: &Path, uid: u32) -> bool { + std::fs::symlink_metadata(path) + .map(|metadata| { + metadata.is_dir() + && !metadata.file_type().is_symlink() + && metadata.uid() == uid + && metadata.mode() & 0o777 == 0o700 + }) + .unwrap_or(false) } pub async fn send_command(path: impl AsRef, command: &str) -> io::Result { let mut stream = UnixStream::connect(path).await?; diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..b8f31b3 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,12 @@ +//! Reusable library components for the `wgcloak` binary. + +pub mod cli; +pub mod client; +pub mod config; +pub mod control; +pub mod datagram; +pub mod protocol; +pub mod server; +pub mod session; +pub mod stats; +pub mod transport; diff --git a/src/main.rs b/src/main.rs index a29fa22..a01e30c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,19 +1,11 @@ -mod cli; -pub mod config; -pub mod control; -pub mod datagram; -pub mod session; -pub mod stats; - -pub mod client; -pub mod protocol; -pub mod server; -pub mod transport; - use anyhow::Result; use clap::Parser; -use cli::{Cli, Command}; -use config::Config; +use wgcloak::{ + cli::{Cli, Command}, + client, + config::{self, Config}, + control, server, +}; #[tokio::main] async fn main() -> Result<()> { @@ -22,12 +14,14 @@ async fn main() -> Result<()> { match cli.command { Command::Client { config } => { let parsed = Config::from_file(&config)?; + parsed.validate().map_err(anyhow::Error::msg)?; let client = parsed.into_client().map_err(anyhow::Error::msg)?; config::init_tracing(&client.logging.level)?; client::run(client).await } Command::Server { config } => { let parsed = Config::from_file(&config)?; + parsed.validate().map_err(anyhow::Error::msg)?; let server = parsed.into_server().map_err(anyhow::Error::msg)?; config::init_tracing(&server.logging.level)?; server::run(server).await diff --git a/src/protocol/auth.rs b/src/protocol/auth.rs index a939421..5e1a9bd 100644 --- a/src/protocol/auth.rs +++ b/src/protocol/auth.rs @@ -186,6 +186,10 @@ pub fn data_auth_tag( tag } +// Each argument is an explicit field in the authenticated wire input. Keep +// this helper aligned with `data_auth_tag` so callers cannot accidentally +// verify a different field set. +#[allow(clippy::too_many_arguments)] pub fn verify_data_auth_tag( session_key: &[u8], version: u8, diff --git a/src/protocol/direct.rs b/src/protocol/direct.rs index 04da8c0..6d69a49 100644 --- a/src/protocol/direct.rs +++ b/src/protocol/direct.rs @@ -81,6 +81,8 @@ pub enum ProtocolError { Truncated { needed: usize, actual: usize }, #[error("packet exceeds maximum size of {maximum} bytes")] Oversized { maximum: usize }, + #[error("DATA payload is {actual} bytes, maximum is {maximum}")] + OversizedDataPayload { actual: usize, maximum: usize }, #[error("unsupported protocol version {0}")] UnsupportedVersion(u8), #[error("unknown packet type 0x{0:02x}")] @@ -635,9 +637,27 @@ fn array(input: &[u8]) -> [u8; N] { output } -/// Parse one complete direct UDP datagram. All structural checks occur before -/// the only potentially large allocation (copying a valid DATA payload). +/// Parse one complete direct UDP datagram without imposing a transport payload +/// limit. All structural checks occur before copying a valid DATA payload. pub fn parse_packet(input: &[u8]) -> Result { + parse_packet_inner(input, None) +} + +/// Parse one packet while bounding DATA payload size before copying it into a +/// `Bytes` value. Public listeners should use this with their negotiated or +/// configured maximum so unauthenticated oversized packets do not allocate a +/// second large buffer. +pub fn parse_packet_bounded( + input: &[u8], + maximum_data_payload: usize, +) -> Result { + parse_packet_inner(input, Some(maximum_data_payload)) +} + +fn parse_packet_inner( + input: &[u8], + maximum_data_payload: Option, +) -> Result { if input.len() > MAX_PACKET_SIZE { return Err(ProtocolError::Oversized { maximum: MAX_PACKET_SIZE, @@ -778,8 +798,15 @@ pub fn parse_packet(input: &[u8]) -> Result { } let sequence = u32::from_be_bytes(array(take(input, &mut at, 4)?)); let auth_tag = array(take(input, &mut at, 16)?); - // The size check above makes this bound safe and caps the copy to - // MAX_PACKET_SIZE - DATA_HEADER_LEN bytes. + let payload_len = input.len() - at; + if let Some(maximum) = maximum_data_payload { + if payload_len > maximum { + return Err(ProtocolError::OversizedDataPayload { + actual: payload_len, + maximum, + }); + } + } let payload = Bytes::copy_from_slice(&input[at..]); Ok(Packet::Data(DataPacket { version, @@ -919,6 +946,53 @@ mod tests { )); } + #[test] + fn bounded_parser_rejects_large_data_before_payload_copy() { + let key = auth::derive_session_key(b"secret", &[1; 16], &[2; 16], 7); + let payload_len = MAX_PACKET_SIZE - DATA_HEADER_LEN; + let packet = DataPacket::authenticated(&key, 7, 1, vec![0u8; payload_len]); + let encoded = packet.encode().unwrap(); + assert_eq!( + parse_packet_bounded(&encoded, 1200), + Err(ProtocolError::OversizedDataPayload { + actual: payload_len, + maximum: 1200, + }) + ); + assert!(matches!( + parse_packet_bounded(&encoded, payload_len), + Ok(Packet::Data(data)) if data.payload.len() == payload_len + )); + } + + #[test] + fn parsed_flags_are_part_of_data_family_authentication() { + let key = auth::derive_session_key(b"secret", &[1; 16], &[2; 16], 7); + let data = DataPacket::authenticated(&key, 7, 1, Bytes::from_static(b"data")); + let mut encoded = data.encode().unwrap(); + encoded[2] = 1; + let Packet::Data(parsed) = parse_packet(&encoded).unwrap() else { + panic!("expected DATA packet"); + }; + assert!(!parsed.verify_authenticator(&key)); + + let keepalive = SessionPacket::authenticated(PacketType::Keepalive, &key, 7, 2); + let mut encoded = keepalive.encode().unwrap(); + encoded[2] = 1; + let Packet::Keepalive(parsed) = parse_packet(&encoded).unwrap() else { + panic!("expected KEEPALIVE packet"); + }; + assert!(!parsed.verify_authenticator(&key)); + + let close = SessionPacket::authenticated(PacketType::Close, &key, 7, 3); + let mut encoded = close.encode().unwrap(); + encoded[2] = 1; + let Packet::Close(parsed) = parse_packet(&encoded).unwrap() else { + panic!("expected CLOSE packet"); + }; + assert!(!parsed.verify_authenticator(&key)); + } + #[test] fn hello_header_length_bound_is_enforced() { assert!(HelloPacket::new(vec![0; MAX_CLIENT_ID_LEN], 0, [0; 16], [0; 32]).is_ok()); diff --git a/src/server/limits.rs b/src/server/limits.rs new file mode 100644 index 0000000..1f4a09c --- /dev/null +++ b/src/server/limits.rs @@ -0,0 +1,3 @@ +//! Server-facing names for the shared transport admission permits. + +pub use crate::transport::{AdmissionController, AdmissionError, PendingHandshake, SessionPermit}; diff --git a/src/server/mod.rs b/src/server/mod.rs index 6f0de81..c8ce095 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,93 +1,602 @@ -use std::sync::Arc; +use std::{ + net::SocketAddr, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; use anyhow::Result; -use tokio::net::UdpSocket; +use bytes::Bytes; +use tokio::{ + net::UdpSocket, + task::JoinSet, + time::{self, timeout}, +}; use tracing::warn; use crate::{ config::ServerConfig, control::{ControlServer, ControlState}, + stats::Stats, transport::{ - quic::QuicServer, DirectUdpServer, TlsTcpServer, TransportError, TransportSession, + quic::QuicServer, AdmissionController, DirectUdpServer, Http3Server, TlsTcpServer, + TransportError, TransportSession, }, }; -/// Run one configured server listener. Direct UDP remains the default; set it -/// off when selecting QUIC or TLS/TCP because each role owns its listener port. +pub mod limits; + +/// Run one configured server listener. Each accepted authenticated session is +/// owned by a JoinSet so listener shutdown can cancel and reap it deterministically. pub async fn run(config: ServerConfig) -> Result<()> { - if config.transports.direct_udp.enabled { - let state = ControlState::default(); - start_control(state.clone()).await?; - return DirectUdpServer::bind(config).await?.run().await; - } - let state = ControlState::default(); - start_control(state.clone()).await?; - if config.transports.quic.enabled { - return run_quic(config, state).await; - } - if config.transports.tls_tcp.enabled { - return run_tls(config, state).await; - } - anyhow::bail!("no server transport enabled") + run_with_control_path(config, crate::control::default_socket_path()).await } -async fn start_control(state: ControlState) -> Result<()> { - let control = ControlServer::bind(crate::control::default_socket_path(), state).await?; - tokio::spawn(async move { - let _ = control.run().await; - }); +/// Runtime entrypoint with an explicit local control path. Keeping the path +/// injectable also allows lifecycle integration tests to use an isolated +/// temporary socket rather than the user's default daemon socket. +pub async fn run_with_control_path( + config: ServerConfig, + control_path: impl Into, +) -> Result<()> { + config.validate()?; + let state = ControlState::default(); + let control = ControlServer::bind(control_path, state.clone()).await?; + let mut control_task = tokio::spawn(control.run()); + + let result = run_listener(config, state.clone()).await; + state.set_status("stopping").await; + state.shutdown.cancel(); + if timeout(Duration::from_secs(2), &mut control_task) + .await + .is_err() + { + control_task.abort(); + } + state.set_status("stopped").await; + result +} + +async fn run_listener(config: ServerConfig, state: ControlState) -> Result<()> { + validate_udp_listener_conflicts(&config)?; + let backend = config.backend.target.parse::()?; + let mut tasks = JoinSet::new(); + let mut transports = Vec::new(); + let direct_active = Arc::new(AtomicUsize::new(0)); + let quic_active = Arc::new(AtomicUsize::new(0)); + let http3_active = Arc::new(AtomicUsize::new(0)); + let tls_active = Arc::new(AtomicUsize::new(0)); + let admission = AdmissionController::new(&config.limits); + + if config.transports.direct_udp.enabled { + let listener = DirectUdpServer::bind(config.clone()).await?; + transports.push("direct_udp"); + let state = state.clone(); + let active = direct_active.clone(); + tasks.spawn(async move { run_direct(listener, state, active).await }); + } + if config.transports.quic.enabled { + let listener = Arc::new(QuicServer::bind_with_admission(&config, admission.clone()).await?); + transports.push("quic"); + let state = state.clone(); + let active = quic_active.clone(); + tasks.spawn(async move { run_quic(listener, backend, state, active).await }); + } + if config.transports.http3.enabled { + let listener = + Arc::new(Http3Server::bind_with_admission(&config, admission.clone()).await?); + transports.push("http3"); + let state = state.clone(); + let active = http3_active.clone(); + tasks.spawn(async move { run_http3(listener, backend, state, active).await }); + } + if config.transports.tls_tcp.enabled { + let listener = + Arc::new(TlsTcpServer::bind_with_admission(&config, admission.clone()).await?); + transports.push("tls_tcp"); + let state = state.clone(); + let active = tls_active.clone(); + tasks.spawn(async move { run_tls(listener, backend, state, active).await }); + } + if tasks.is_empty() { + anyhow::bail!("no server transport enabled") + } + + let transport_names = transports.join(","); + state + .set_status(format!("listening; transports={transport_names}")) + .await; + let signal = shutdown_signal(); + tokio::pin!(signal); + let first_result = tokio::select! { + _ = state.shutdown.cancelled() => Ok(()), + _ = &mut signal => Ok(()), + result = tasks.join_next() => match result { + Some(Ok(result)) => result, + Some(Err(error)) => Err(anyhow::anyhow!("transport listener task failed: {error}")), + None => Ok(()), + }, + }; + // Each carrier observes this shared token and performs its own bounded + // session drain before the final abort fallback below. + state.shutdown.cancel(); + let drain = async { + while let Some(result) = tasks.join_next().await { + match result { + Ok(Ok(())) => {} + Ok(Err(error)) => warn!(%error, "transport listener stopped with an error"), + Err(error) => warn!(%error, "transport listener task did not join cleanly"), + } + } + }; + if timeout(Duration::from_secs(4), drain).await.is_err() { + tasks.abort_all(); + while tasks.join_next().await.is_some() {} + } + first_result +} + +fn validate_udp_listener_conflicts(config: &ServerConfig) -> Result<()> { + let udp = [ + ( + "direct_udp", + config.transports.direct_udp.enabled, + config.transports.direct_udp.port, + ), + ( + "quic", + config.transports.quic.enabled, + config.transports.quic.port, + ), + ( + "http3", + config.transports.http3.enabled, + config.transports.http3.port, + ), + ]; + for (index, (left_name, left_enabled, left_port)) in udp.iter().enumerate() { + if !*left_enabled { + continue; + } + for (right_name, right_enabled, right_port) in udp.iter().skip(index + 1) { + if *right_enabled && left_port == right_port { + anyhow::bail!( + "UDP listeners {left_name} and {right_name} both bind {}:{left_port}; configure distinct UDP ports", + config.server.listen + ); + } + } + } Ok(()) } -async fn run_quic(config: ServerConfig, state: ControlState) -> Result<()> { - let listener = Arc::new(QuicServer::bind(&config).await?); - let backend = config.backend.target.parse()?; +async fn run_direct( + listener: DirectUdpServer, + state: ControlState, + active_count: Arc, +) -> Result<()> { + let observer = listener.handle(); + let stats = observer.stats(); + let mut refresh = time::interval(Duration::from_secs(1)); + let run = listener.run_with_shutdown(state.shutdown.clone()); + tokio::pin!(run); loop { tokio::select! { - _ = state.shutdown.cancelled() => { listener.close(); return Ok(()); } - result = listener.accept_session() => match result { - Ok(session) => { let state = state.clone(); tokio::spawn(async move { let _ = relay_session(Box::new(session), backend, state).await; }); } - Err(TransportError::AuthenticationRejected(_)) => warn!("QUIC client authentication rejected"), - Err(error) => warn!(error=%error, "QUIC session failed"), + result = &mut run => { + let active = observer.active_sessions().await; + active_count.store(active, Ordering::Relaxed); + publish_server_stats_with_client_ids( + &state, + &stats, + "direct_udp", + active, + observer.active_sessions_by_client_id().await, + ) + .await; + return result; } + _ = refresh.tick() => { + let active = observer.active_sessions().await; + active_count.store(active, Ordering::Relaxed); + publish_server_stats_with_client_ids( + &state, + &stats, + "direct_udp", + active, + observer.active_sessions_by_client_id().await, + ).await; + }, } } } -async fn run_tls(config: ServerConfig, state: ControlState) -> Result<()> { - let listener = Arc::new(TlsTcpServer::bind(&config).await?); - let backend = config.backend.target.parse()?; +async fn run_http3( + listener: Arc, + backend: SocketAddr, + state: ControlState, + active: Arc, +) -> Result<()> { + let stats = Arc::new(Stats::default()); + let mut tasks = JoinSet::new(); + let mut handshakes: JoinSet< + Result, + > = JoinSet::new(); + let mut refresh = time::interval(Duration::from_secs(1)); loop { tokio::select! { - _ = state.shutdown.cancelled() => return Ok(()), - result = listener.accept_session() => match result { - Ok(session) => { let state = state.clone(); tokio::spawn(async move { let _ = relay_session(session, backend, state).await; }); } - Err(error) => warn!(error=%error, "TLS/TCP session failed"), + _ = state.shutdown.cancelled() => { + listener.close(); + break; + } + _ = refresh.tick() => { + publish_server_stats(&state, &stats, "http3", Some(active.load(Ordering::Relaxed))).await; + } + Some(result) = tasks.join_next(), if !tasks.is_empty() => { + if let Err(error) = result { warn!(%error, "HTTP/3 session task failed"); } + } + Some(result) = handshakes.join_next(), if !handshakes.is_empty() => { + match result { + Ok(Ok(session)) => { + let client_session = session + .client_id() + .map(|client_id| stats.track_client_session(client_id.to_owned())); + let state = state.clone(); + let stats = stats.clone(); + let active = active.clone(); + active.fetch_add(1, Ordering::Relaxed); + tasks.spawn(async move { + let _guard = ActiveSessionGuard(active); + let _client_session = client_session; + if let Err(error) = relay_session(Box::new(session), backend, state, stats.clone()).await { + stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + warn!(%error, "HTTP/3 session relay ended with an error"); + } + }); + } + Ok(Err(TransportError::AuthenticationRejected(_))) => { + stats.authentication_rejections.fetch_add(1, Ordering::Relaxed); + stats.authentication_failures.fetch_add(1, Ordering::Relaxed); + warn!("HTTP/3 client authentication rejected"); + } + Ok(Err(error)) => log_admission_failure("HTTP/3", error), + Err(error) => warn!(%error, "HTTP/3 accept task failed"), + } + } + pending = listener.accept_pending() => { + stats.handshake_attempts.fetch_add(1, Ordering::Relaxed); + match pending { + Ok(pending) => { + let listener = listener.clone(); + handshakes.spawn(async move { listener.authenticate_pending(pending).await }); + } + Err(error) => log_admission_failure("HTTP/3", error), + } } } } + handshakes.abort_all(); + while handshakes.join_next().await.is_some() {} + drain_sessions(&mut tasks).await; + publish_server_stats( + &state, + &stats, + "http3", + Some(active.load(Ordering::Relaxed)), + ) + .await; + Ok(()) +} + +async fn run_quic( + listener: Arc, + backend: SocketAddr, + state: ControlState, + active: Arc, +) -> Result<()> { + let stats = Arc::new(Stats::default()); + let mut tasks = JoinSet::new(); + let mut handshakes: JoinSet< + Result, + > = JoinSet::new(); + let mut refresh = time::interval(Duration::from_secs(1)); + loop { + tokio::select! { + _ = state.shutdown.cancelled() => { + listener.close(); + break; + } + _ = refresh.tick() => { + publish_server_stats(&state, &stats, "quic", Some(active.load(Ordering::Relaxed))).await + }, + Some(result) = tasks.join_next(), if !tasks.is_empty() => { + if let Err(error) = result { warn!(%error, "QUIC session task failed"); } + } + Some(result) = handshakes.join_next(), if !handshakes.is_empty() => { + match result { + Ok(Ok(session)) => { + let client_session = session + .client_id() + .map(|client_id| stats.track_client_session(client_id.to_owned())); + let state = state.clone(); + let stats = stats.clone(); + let active = active.clone(); + let _ = active.fetch_add(1, Ordering::Relaxed); + tasks.spawn(async move { + let _guard = ActiveSessionGuard(active); + let _client_session = client_session; + if let Err(error) = relay_session(Box::new(session), backend, state, stats.clone()).await { + stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + warn!(%error, "QUIC session relay ended with an error"); + } + }); + } + Ok(Err(TransportError::AuthenticationRejected(_))) => { + stats.authentication_rejections.fetch_add(1, Ordering::Relaxed); + stats.authentication_failures.fetch_add(1, Ordering::Relaxed); + warn!("QUIC client authentication rejected"); + } + Ok(Err(error)) => log_admission_failure("QUIC", error), + Err(error) => warn!(%error, "QUIC accept task failed"), + } + } + pending = listener.accept_pending() => { + stats.handshake_attempts.fetch_add(1, Ordering::Relaxed); + match pending { + Ok(pending) => { + let listener = listener.clone(); + handshakes.spawn(async move { listener.authenticate_pending(pending).await }); + } + Err(error) => log_admission_failure("QUIC", error), + } + } + } + } + handshakes.abort_all(); + while handshakes.join_next().await.is_some() {} + drain_sessions(&mut tasks).await; + publish_server_stats(&state, &stats, "quic", Some(active.load(Ordering::Relaxed))).await; + Ok(()) +} + +async fn run_tls( + listener: Arc, + backend: SocketAddr, + state: ControlState, + active: Arc, +) -> Result<()> { + let stats = Arc::new(Stats::default()); + let mut tasks = JoinSet::new(); + let mut handshakes: JoinSet, TransportError>> = JoinSet::new(); + let mut refresh = time::interval(Duration::from_secs(1)); + loop { + tokio::select! { + _ = state.shutdown.cancelled() => break, + _ = refresh.tick() => { + publish_server_stats(&state, &stats, "tls_tcp", Some(active.load(Ordering::Relaxed))).await + }, + Some(result) = tasks.join_next(), if !tasks.is_empty() => { + if let Err(error) = result { warn!(%error, "TLS/TCP session task failed"); } + } + Some(result) = handshakes.join_next(), if !handshakes.is_empty() => { + match result { + Ok(Ok(session)) => { + let client_session = session + .client_id() + .map(|client_id| stats.track_client_session(client_id.to_owned())); + let state = state.clone(); + let stats = stats.clone(); + let active = active.clone(); + let _ = active.fetch_add(1, Ordering::Relaxed); + tasks.spawn(async move { + let _guard = ActiveSessionGuard(active); + let _client_session = client_session; + if let Err(error) = relay_session(session, backend, state, stats.clone()).await { + stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + warn!(%error, "TLS/TCP session relay ended with an error"); + } + }); + } + Ok(Err(TransportError::AuthenticationRejected(_))) => { + stats.authentication_rejections.fetch_add(1, Ordering::Relaxed); + stats.authentication_failures.fetch_add(1, Ordering::Relaxed); + warn!("TLS/TCP client authentication rejected"); + } + Ok(Err(error)) => log_admission_failure("TLS/TCP", error), + Err(error) => warn!(%error, "TLS/TCP accept task failed"), + } + } + pending = listener.accept_pending() => { + stats.handshake_attempts.fetch_add(1, Ordering::Relaxed); + match pending { + Ok(pending) => { + let listener = listener.clone(); + handshakes.spawn(async move { listener.authenticate_pending(pending).await }); + } + Err(error) => log_admission_failure("TLS/TCP", error), + } + } + } + } + handshakes.abort_all(); + while handshakes.join_next().await.is_some() {} + drain_sessions(&mut tasks).await; + publish_server_stats( + &state, + &stats, + "tls_tcp", + Some(active.load(Ordering::Relaxed)), + ) + .await; + Ok(()) +} + +struct ActiveSessionGuard(Arc); +impl Drop for ActiveSessionGuard { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::Relaxed); + } +} + +fn log_admission_failure(transport: &str, error: TransportError) { + match error { + TransportError::Busy => warn!(transport, "encrypted transport admission capacity reached"), + TransportError::RateLimited => { + warn!( + transport, + "encrypted transport source is temporarily rate limited" + ) + } + TransportError::Closed => {} + error => warn!(transport, %error, "encrypted transport session failed"), + } +} + +async fn drain_sessions(tasks: &mut JoinSet<()>) { + let drain = async { while tasks.join_next().await.is_some() {} }; + if timeout(Duration::from_secs(2), drain).await.is_err() { + tasks.abort_all(); + while tasks.join_next().await.is_some() {} + } } async fn relay_session( session: Box, - backend_target: std::net::SocketAddr, + backend_target: SocketAddr, state: ControlState, + stats: Arc, ) -> Result<()> { - let backend = UdpSocket::bind("0.0.0.0:0").await?; - backend.connect(backend_target).await?; - let backend = Arc::new(backend); + let result = relay_session_inner(session.as_ref(), backend_target, &state, &stats).await; + // TLS/TCP has a bounded close handshake; cap all carriers uniformly so + // session cleanup cannot hold the listener's shutdown path indefinitely. + let _ = timeout(Duration::from_secs(2), session.close()).await; + result +} + +async fn relay_session_inner( + session: &dyn TransportSession, + backend_target: SocketAddr, + state: &ControlState, + stats: &Stats, +) -> Result<()> { + let backend = match UdpSocket::bind("0.0.0.0:0").await { + Ok(socket) => socket, + Err(error) => { + stats.backend_errors.fetch_add(1, Ordering::Relaxed); + return Err(error.into()); + } + }; + if let Err(error) = backend.connect(backend_target).await { + stats.backend_errors.fetch_add(1, Ordering::Relaxed); + return Err(error.into()); + } let mut buf = vec![0u8; session.max_datagram_size().saturating_add(1)]; loop { tokio::select! { - _ = state.shutdown.cancelled() => { let _ = session.close().await; return Ok(()); } + _ = state.shutdown.cancelled() => return Ok(()), received = session.recv_datagram() => { let payload = received?; - if payload.len() <= session.max_datagram_size() { backend.send(&payload).await?; } + if payload.len() > session.max_datagram_size() { + stats.oversized_datagrams.fetch_add(1, Ordering::Relaxed); + stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + continue; + } + match timeout(Duration::from_secs(2), backend.send(&payload)).await { + Ok(Ok(_)) => stats.add_received(payload.len()), + Ok(Err(error)) => { + stats.backend_errors.fetch_add(1, Ordering::Relaxed); + stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + return Err(error.into()); + } + Err(_) => { + stats.backend_errors.fetch_add(1, Ordering::Relaxed); + stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + return Err(anyhow::anyhow!("backend UDP send timed out")); + } + } } received = backend.recv(&mut buf) => { - let len = received?; - session.send_datagram(bytes::Bytes::copy_from_slice(&buf[..len])).await?; + let len = match received { + Ok(len) => len, + Err(error) => { + stats.backend_errors.fetch_add(1, Ordering::Relaxed); + stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + return Err(error.into()); + } + }; + if len > session.max_datagram_size() { + stats.oversized_datagrams.fetch_add(1, Ordering::Relaxed); + stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + continue; + } + match timeout( + Duration::from_secs(2), + session.send_datagram(Bytes::copy_from_slice(&buf[..len])), + ) + .await + { + Ok(Ok(())) => {} + Ok(Err(error)) => { + stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + return Err(error.into()); + } + Err(_) => { + stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + return Err(anyhow::anyhow!("transport datagram send timed out")); + } + } + stats.add_sent(len); } } } } + +async fn publish_server_stats( + state: &ControlState, + stats: &Stats, + transport: &str, + active_sessions: Option, +) { + publish_server_stats_with_client_ids( + state, + stats, + transport, + active_sessions.unwrap_or(0), + stats.active_sessions_by_client_id(), + ) + .await; +} + +async fn publish_server_stats_with_client_ids( + state: &ControlState, + stats: &Stats, + transport: &str, + active_sessions: usize, + active_sessions_by_client_id: std::collections::BTreeMap, +) { + state + .set_transport_stats_by_client_id( + transport, + active_sessions, + stats.snapshot(), + active_sessions_by_client_id, + ) + .await; +} + +async fn shutdown_signal() { + #[cfg(unix)] + { + use tokio::signal::unix::{signal, SignalKind}; + let mut term = signal(SignalKind::terminate()).expect("install SIGTERM handler"); + tokio::select! { + _ = tokio::signal::ctrl_c() => {}, + _ = term.recv() => {}, + } + } + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + } +} diff --git a/src/stats.rs b/src/stats.rs index 489ea69..e0682e8 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,4 +1,10 @@ -use std::sync::atomic::{AtomicU64, Ordering}; +use std::{ + collections::BTreeMap, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, + }, +}; /// Counters shared by a session or runtime. Values are monotonic. #[derive(Debug, Default)] @@ -16,9 +22,32 @@ pub struct Stats { pub authentication_rejections: AtomicU64, pub rate_limited_attempts: AtomicU64, pub backend_errors: AtomicU64, + active_sessions_by_client_id: Arc>>, } impl Stats { + /// Track one authenticated session until its guard is dropped. Counts are + /// deterministic and safe to snapshot while session tasks are exiting. + pub fn track_client_session(&self, client_id: impl Into) -> ActiveClientSession { + let client_id = client_id.into(); + let mut active = self + .active_sessions_by_client_id + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *active.entry(client_id.clone()).or_default() += 1; + ActiveClientSession { + active_sessions: self.active_sessions_by_client_id.clone(), + client_id, + } + } + + pub fn active_sessions_by_client_id(&self) -> BTreeMap { + self.active_sessions_by_client_id + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + pub fn add_sent(&self, bytes: usize) { self.bytes_sent.fetch_add(bytes as u64, Ordering::Relaxed); self.datagrams_sent.fetch_add(1, Ordering::Relaxed); @@ -49,6 +78,26 @@ impl Stats { } } +pub struct ActiveClientSession { + active_sessions: Arc>>, + client_id: String, +} + +impl Drop for ActiveClientSession { + fn drop(&mut self) { + let mut active = self + .active_sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(count) = active.get_mut(&self.client_id) { + *count = count.saturating_sub(1); + if *count == 0 { + active.remove(&self.client_id); + } + } + } +} + #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct StatsSnapshot { pub bytes_sent: u64, @@ -65,3 +114,39 @@ pub struct StatsSnapshot { pub rate_limited_attempts: u64, pub backend_errors: u64, } + +impl StatsSnapshot { + /// Add counters from independent transport listeners without losing any + /// per-listener accounting during a multi-transport server run. + pub fn saturating_add(self, other: Self) -> Self { + Self { + bytes_sent: self.bytes_sent.saturating_add(other.bytes_sent), + bytes_received: self.bytes_received.saturating_add(other.bytes_received), + datagrams_sent: self.datagrams_sent.saturating_add(other.datagrams_sent), + datagrams_received: self + .datagrams_received + .saturating_add(other.datagrams_received), + datagrams_dropped: self + .datagrams_dropped + .saturating_add(other.datagrams_dropped), + oversized_datagrams: self + .oversized_datagrams + .saturating_add(other.oversized_datagrams), + authentication_failures: self + .authentication_failures + .saturating_add(other.authentication_failures), + reconnections: self.reconnections.saturating_add(other.reconnections), + expired_sessions: self.expired_sessions.saturating_add(other.expired_sessions), + handshake_attempts: self + .handshake_attempts + .saturating_add(other.handshake_attempts), + authentication_rejections: self + .authentication_rejections + .saturating_add(other.authentication_rejections), + rate_limited_attempts: self + .rate_limited_attempts + .saturating_add(other.rate_limited_attempts), + backend_errors: self.backend_errors.saturating_add(other.backend_errors), + } + } +} diff --git a/src/transport/direct_udp.rs b/src/transport/direct_udp.rs index 5d02fb0..a685d1b 100644 --- a/src/transport/direct_udp.rs +++ b/src/transport/direct_udp.rs @@ -30,7 +30,7 @@ use crate::{ AcceptPacket, DataPacket, HelloPacket, Packet, PacketType, RejectPacket, RejectReason, SessionPacket, }, - replay::{NonceReplayCache, ReplayWindow}, + replay::ReplayWindow, }, stats::Stats, }; @@ -39,6 +39,8 @@ use super::{TransportConnector, TransportError, TransportFuture, TransportSessio const DEFAULT_MAX_DATAGRAM: usize = 1200; const MAX_UDP_DATAGRAM: usize = u16::MAX as usize; +const HELLO_REPLAY_WINDOW: Duration = Duration::from_secs(120); +const MAX_HELLO_REPLAY_ENTRIES: usize = 4096; fn unix_timestamp() -> u64 { SystemTime::now() @@ -71,10 +73,12 @@ pub struct DirectUdpSession { socket: Arc, peer: Mutex, session_id: u64, + client_id: String, session_key: [u8; auth::SESSION_KEY_LEN], send_sequence: AtomicU32, receive_window: Mutex, max_datagram: usize, + keepalive_interval: Option, closed: CancellationToken, } @@ -147,10 +151,16 @@ impl DirectUdpSession { socket, peer: Mutex::new(source), session_id: accept.session_id, + client_id: config.client.id.clone(), session_key, send_sequence: AtomicU32::new(auth::generate_sequence_number()), receive_window: Mutex::new(ReplayWindow::new()), max_datagram, + keepalive_interval: (config.transport.direct_udp.keepalive_secs + > 0) + .then(|| { + Duration::from_secs(config.transport.direct_udp.keepalive_secs) + }), closed: CancellationToken::new(), }); } @@ -193,13 +203,14 @@ impl DirectUdpSession { let mut buf = vec![0u8; MAX_UDP_DATAGRAM]; loop { let (len, source) = self.socket.recv_from(&mut buf).await?; - let packet = match protocol::parse_packet(&buf[..len]) { - Ok(p) => p, - Err(e) => { - debug!(error=%e, "dropping malformed direct UDP packet"); - continue; - } - }; + let packet = + match protocol::direct::parse_packet_bounded(&buf[..len], self.max_datagram) { + Ok(p) => p, + Err(e) => { + debug!(error=%e, "dropping malformed direct UDP packet"); + continue; + } + }; let valid = match &packet { Packet::Data(p) => { p.session_id == self.session_id && p.verify_authenticator(&self.session_key) @@ -275,6 +286,18 @@ impl TransportSession for DirectUdpSession { fn max_datagram_size(&self) -> usize { self.max_datagram } + + fn client_id(&self) -> Option<&str> { + Some(&self.client_id) + } + + fn send_keepalive<'a>(&'a self) -> TransportFuture<'a, ()> { + Box::pin(async move { DirectUdpSession::send_keepalive(self).await }) + } + + fn authenticated_keepalive_interval(&self) -> Option { + self.keepalive_interval + } } pub struct DirectUdpConnector { @@ -297,6 +320,7 @@ impl TransportConnector for DirectUdpConnector { struct ServerSession { id: u64, client_id: String, + hello_key: HelloCacheKey, key: [u8; auth::SESSION_KEY_LEN], peer: Mutex, backend: Arc, @@ -307,6 +331,110 @@ struct ServerSession { cancel: CancellationToken, } +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct HelloCacheKey { + client_id: Vec, + client_nonce: [u8; 16], +} + +impl HelloCacheKey { + fn new(hello: &HelloPacket) -> Self { + Self { + client_id: hello.client_id.to_vec(), + client_nonce: hello.client_nonce, + } + } +} + +struct HelloCacheEntry { + hello: HelloPacket, + peer: SocketAddr, + session_id: Option, + accept_bytes: Option>, + expires_at: Instant, +} + +enum HelloCacheLookup { + New, + Retry { + session_id: u64, + accept_bytes: Vec, + }, + Replay, + Full, +} + +/// Bounded replay cache for accepted HELLOs. Active entries retain the original +/// ACCEPT so a retransmission can recover a lost response; closed/expired +/// sessions become nonce tombstones until the replay window elapses. +#[derive(Default)] +struct HelloReplayCache { + entries: HashMap, +} + +impl HelloReplayCache { + fn purge_expired(&mut self, now: Instant) { + self.entries.retain(|_, entry| entry.expires_at > now); + } + + fn lookup(&mut self, hello: &HelloPacket, peer: SocketAddr, now: Instant) -> HelloCacheLookup { + self.purge_expired(now); + let key = HelloCacheKey::new(hello); + if let Some(entry) = self.entries.get(&key) { + if entry.hello != *hello || entry.peer != peer { + return HelloCacheLookup::Replay; + } + return match (entry.session_id, entry.accept_bytes.as_ref()) { + (Some(session_id), Some(accept_bytes)) => HelloCacheLookup::Retry { + session_id, + accept_bytes: accept_bytes.clone(), + }, + _ => HelloCacheLookup::Replay, + }; + } + if self.entries.len() >= MAX_HELLO_REPLAY_ENTRIES { + HelloCacheLookup::Full + } else { + HelloCacheLookup::New + } + } + + fn insert( + &mut self, + hello: HelloPacket, + peer: SocketAddr, + session_id: u64, + accept_bytes: Vec, + now: Instant, + ) -> bool { + self.purge_expired(now); + let key = HelloCacheKey::new(&hello); + if self.entries.contains_key(&key) || self.entries.len() >= MAX_HELLO_REPLAY_ENTRIES { + return false; + } + self.entries.insert( + key, + HelloCacheEntry { + hello, + peer, + session_id: Some(session_id), + accept_bytes: Some(accept_bytes), + expires_at: now + HELLO_REPLAY_WINDOW, + }, + ); + true + } + + fn deactivate(&mut self, key: &HelloCacheKey, session_id: u64) { + if let Some(entry) = self.entries.get_mut(key) { + if entry.session_id == Some(session_id) { + entry.session_id = None; + entry.accept_bytes = None; + } + } + } +} + /// Multi-client direct UDP server. Each authenticated session gets its own /// connected backend socket and therefore natural response demultiplexing. pub struct DirectUdpServer { @@ -314,12 +442,40 @@ pub struct DirectUdpServer { backend_target: SocketAddr, clients: HashMap>, sessions: Arc>>>, - nonces: Arc>, + hello_cache: Mutex, pending: Arc>>, config: ServerConfig, stats: Arc, } +/// Read-only observation handle that remains usable after `run(self)` takes +/// ownership of the listener. The session registry stays owned by the server; +/// observers only share its lock and aggregate counters. +#[derive(Clone)] +pub struct DirectUdpServerHandle { + sessions: Arc>>>, + stats: Arc, +} + +impl DirectUdpServerHandle { + pub async fn active_sessions(&self) -> usize { + self.sessions.lock().await.len() + } + + pub fn stats(&self) -> Arc { + self.stats.clone() + } + + pub async fn active_sessions_by_client_id(&self) -> std::collections::BTreeMap { + let sessions = self.sessions.lock().await; + let mut active = std::collections::BTreeMap::new(); + for session in sessions.values() { + *active.entry(session.client_id.clone()).or_default() += 1; + } + active + } +} + impl DirectUdpServer { pub async fn bind(config: ServerConfig) -> Result { let bind = direct_bind_addr(&config.server.listen, config.transports.direct_udp.port)?; @@ -347,7 +503,7 @@ impl DirectUdpServer { backend_target, clients, sessions: Arc::new(Mutex::new(HashMap::new())), - nonces: Arc::new(Mutex::new(NonceReplayCache::new(Duration::from_secs(120)))), + hello_cache: Mutex::new(HelloReplayCache::default()), pending: Arc::new(Mutex::new(HashMap::new())), config, stats: Arc::new(Stats::default()), @@ -359,6 +515,14 @@ impl DirectUdpServer { pub fn stats(&self) -> Arc { self.stats.clone() } + + pub fn handle(&self) -> DirectUdpServerHandle { + DirectUdpServerHandle { + sessions: self.sessions.clone(), + stats: self.stats.clone(), + } + } + pub async fn active_sessions(&self) -> usize { self.sessions.lock().await.len() } @@ -400,26 +564,6 @@ impl DirectUdpServer { } pending.insert(peer, now); } - let source_ip = peer.ip(); - let active = self.sessions.lock().await; - let sessions_for_ip = { - let mut count = 0; - for session in active.values() { - if session.peer.lock().await.ip() == source_ip { - count += 1; - } - } - count - }; - if active.len() >= self.config.limits.max_sessions - || sessions_for_ip >= self.config.limits.max_sessions_per_ip - { - drop(active); - self.pending.lock().await.remove(&peer); - self.send_reject(peer, RejectReason::ServerBusy).await; - return Ok(None); - } - drop(active); let Some(secret) = self .clients .get(std::str::from_utf8(&hello.client_id).unwrap_or("")) @@ -429,9 +573,7 @@ impl DirectUdpServer { .await; return Ok(None); }; - if !hello.is_fresh(SystemTime::now(), Duration::from_secs(60)) - || !hello.verify_authenticator(secret) - { + if !hello.verify_authenticator(secret) { self.stats .authentication_failures .fetch_add(1, Ordering::Relaxed); @@ -440,22 +582,89 @@ impl DirectUdpServer { .await; return Ok(None); } - if !self.nonces.lock().await.check_and_record( - &hello.client_id, - hello.client_nonce, - Instant::now(), - ) { + + let now = Instant::now(); + let cache_lookup = self.hello_cache.lock().await.lookup(&hello, peer, now); + match cache_lookup { + HelloCacheLookup::Retry { + session_id, + accept_bytes, + } => { + self.pending.lock().await.remove(&peer); + let active = self.sessions.lock().await.contains_key(&session_id); + if !active { + self.hello_cache + .lock() + .await + .deactivate(&HelloCacheKey::new(&hello), session_id); + self.send_reject(peer, RejectReason::ReplayDetected).await; + return Ok(None); + } + match self.socket.send_to(&accept_bytes, peer).await { + Ok(_) => self.stats.add_sent(accept_bytes.len()), + Err(error) => { + self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + warn!(remote_addr=%peer, %error, "failed to retransmit cached direct UDP ACCEPT") + } + } + // The session task already exists; a retry only resends the + // original response and must not start another backend loop. + return Ok(None); + } + HelloCacheLookup::Replay => { + self.pending.lock().await.remove(&peer); + self.send_reject(peer, RejectReason::ReplayDetected).await; + return Ok(None); + } + HelloCacheLookup::Full => { + self.pending.lock().await.remove(&peer); + self.stats + .rate_limited_attempts + .fetch_add(1, Ordering::Relaxed); + self.send_reject(peer, RejectReason::RateLimited).await; + return Ok(None); + } + HelloCacheLookup::New => {} + } + + // Exact authenticated retries reuse the original response for the + // entire replay-cache window, even after the initial timestamp skew + // window has elapsed. A new nonce still must be fresh before allocation. + if !hello.is_fresh(SystemTime::now(), Duration::from_secs(60)) { + self.stats + .authentication_failures + .fetch_add(1, Ordering::Relaxed); self.pending.lock().await.remove(&peer); - self.send_reject(peer, RejectReason::ReplayDetected).await; + self.send_reject(peer, RejectReason::AuthenticationFailed) + .await; return Ok(None); } + + let source_ip = peer.ip(); + let active = self.sessions.lock().await; + let mut sessions_for_ip = 0; + for session in active.values() { + if session.peer.lock().await.ip() == source_ip { + sessions_for_ip += 1; + } + } + let at_limit = active.len() >= self.config.limits.max_sessions + || sessions_for_ip >= self.config.limits.max_sessions_per_ip; + drop(active); + if at_limit { + self.pending.lock().await.remove(&peer); + self.send_reject(peer, RejectReason::ServerBusy).await; + return Ok(None); + } + let backend = Arc::new(UdpSocket::bind("0.0.0.0:0").await?); backend.connect(self.backend_target).await?; let mut session_id = auth::generate_session_id(); - let mut sessions = self.sessions.lock().await; + let sessions = self.sessions.lock().await; while sessions.contains_key(&session_id) { session_id = auth::generate_session_id(); } + drop(sessions); let mut server_nonce = [0u8; 16]; OsRng.fill_bytes(&mut server_nonce); let idle = self.config.session.idle_timeout_secs.min(u32::MAX as u64) as u32; @@ -464,6 +673,7 @@ impl DirectUdpServer { let session = Arc::new(ServerSession { id: session_id, client_id: String::from_utf8_lossy(&hello.client_id).into_owned(), + hello_key: HelloCacheKey::new(&hello), key, peer: Mutex::new(peer), backend, @@ -473,9 +683,6 @@ impl DirectUdpServer { max_datagram: max_datagram as usize, cancel: CancellationToken::new(), }); - sessions.insert(session_id, session.clone()); - drop(sessions); - self.pending.lock().await.remove(&peer); let accept = AcceptPacket::authenticated( secret, session_id, @@ -485,8 +692,29 @@ impl DirectUdpServer { max_datagram, ); let accept_bytes = accept.encode()?; - self.socket.send_to(&accept_bytes, peer).await?; - self.stats.add_sent(accept_bytes.len()); + if !self + .hello_cache + .lock() + .await + .insert(hello, peer, session_id, accept_bytes.clone(), now) + { + self.send_reject(peer, RejectReason::RateLimited).await; + return Ok(None); + } + self.sessions + .lock() + .await + .insert(session_id, session.clone()); + self.pending.lock().await.remove(&peer); + // Keep and run the authenticated session even if this send fails. The + // client retries the exact HELLO, which can now recover this ACCEPT. + match self.socket.send_to(&accept_bytes, peer).await { + Ok(_) => self.stats.add_sent(accept_bytes.len()), + Err(error) => { + self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + warn!(session_id, remote_addr=%peer, %error, "failed to send direct UDP ACCEPT; retaining session for retry") + } + } info!(session_id, client_id=%session.client_id, remote_addr=%peer, "direct UDP client authenticated"); Ok(Some(session)) } @@ -494,13 +722,28 @@ impl DirectUdpServer { async fn remove_session(&self, id: u64, expired: bool) { if let Some(session) = self.sessions.lock().await.remove(&id) { session.cancel.cancel(); + self.hello_cache + .lock() + .await + .deactivate(&session.hello_key, id); if expired { self.stats.expired_sessions.fetch_add(1, Ordering::Relaxed); } } } + /// Run until SIGINT/SIGTERM, then cancel sessions and await their cleanup. pub async fn run(self) -> Result<()> { + self.run_inner(None).await + } + + /// Runtime-owned cancellation path used by the server supervisor. Both + /// this and [`Self::run`] use the same explicit session cleanup sequence. + pub async fn run_with_shutdown(self, shutdown: CancellationToken) -> Result<()> { + self.run_inner(Some(shutdown)).await + } + + async fn run_inner(self, shutdown: Option) -> Result<()> { let mut tasks = JoinSet::new(); let mut buf = vec![0u8; MAX_UDP_DATAGRAM]; let mut cleanup = time::interval(Duration::from_secs(1)); @@ -509,16 +752,36 @@ impl DirectUdpServer { loop { tokio::select! { _ = &mut signal => { info!("direct UDP server shutting down"); break; } + _ = async { + match shutdown.as_ref() { + Some(token) => token.cancelled().await, + None => std::future::pending().await, + } + } => { info!("direct UDP server cancellation requested"); break; } Some(done) = tasks.join_next() => { if let Ok(id) = done { self.remove_session(id, false).await; } } _ = cleanup.tick() => { let now = Instant::now(); + self.hello_cache.lock().await.purge_expired(now); let expired: Vec = self.sessions.lock().await.iter().filter_map(|(&id,s)| async_ready_expired(s, now, Duration::from_secs(self.config.session.idle_timeout_secs)).map(|_| id)).collect(); for id in expired { self.remove_session(id, true).await; } } received = self.socket.recv_from(&mut buf) => { let (len, peer) = match received { Ok(v) => v, Err(e) => { warn!(error=%e, "direct UDP receive failed"); continue; } }; self.stats.add_received(len); - let packet = match protocol::parse_packet(&buf[..len]) { Ok(p) => p, Err(e) => { self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); debug!(remote_addr=%peer,error=%e,"dropping malformed direct UDP packet"); continue; } }; + let packet = match protocol::direct::parse_packet_bounded(&buf[..len], DEFAULT_MAX_DATAGRAM) { + Ok(p) => p, + Err(protocol::direct::ProtocolError::OversizedDataPayload { actual, maximum }) => { + self.stats.oversized_datagrams.fetch_add(1, Ordering::Relaxed); + self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + debug!(remote_addr=%peer, actual, maximum, "dropping oversized direct UDP DATA before allocation"); + continue; + } + Err(e) => { + self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + debug!(remote_addr=%peer,error=%e,"dropping malformed direct UDP packet"); + continue; + } + }; match packet { Packet::Hello(hello) => match self.handle_hello(hello, peer).await { Ok(Some(session)) => { @@ -528,10 +791,33 @@ impl DirectUdpServer { Ok(None) => {} Err(error) => warn!(remote_addr=%peer, error=%error, "failed to establish direct UDP session"), }, - Packet::Data(data) => if let Err(error) = self.handle_session_packet(PacketType::Data, data.session_id, data.sequence, data.auth_tag, data.payload, peer).await { self.stats.backend_errors.fetch_add(1, Ordering::Relaxed); self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); warn!(session_id=data.session_id, error=%error, "backend forwarding failed"); }, - Packet::Keepalive(p) => if let Err(error) = self.handle_session_packet(p.packet_type, p.session_id, p.sequence, p.auth_tag, Bytes::new(), peer).await { warn!(session_id=p.session_id, error=%error, "keepalive handling failed"); }, - Packet::Close(p) => { if let Err(error) = self.handle_session_packet(p.packet_type, p.session_id, p.sequence, p.auth_tag, Bytes::new(), peer).await { warn!(session_id=p.session_id, error=%error, "close handling failed"); } self.remove_session(p.session_id, false).await; }, - _ => {} + Packet::Data(data) => { + let session_id = data.session_id; + if let Err(error) = self.handle_session_packet(Packet::Data(data), peer).await { + self.stats.backend_errors.fetch_add(1, Ordering::Relaxed); + self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + warn!(session_id, error=%error, "backend forwarding failed"); + } + } + Packet::Keepalive(packet) => { + let session_id = packet.session_id; + if let Err(error) = self.handle_session_packet(Packet::Keepalive(packet), peer).await { + self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + warn!(session_id, error=%error, "keepalive handling failed"); + } + } + Packet::Close(packet) => { + let session_id = packet.session_id; + match self.handle_session_packet(Packet::Close(packet), peer).await { + Ok(true) => self.remove_session(session_id, false).await, + Ok(false) => {}, + Err(error) => { + self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + warn!(session_id, error=%error, "close handling failed"); + } + } + } + _ => { self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); } } } } @@ -540,24 +826,44 @@ impl DirectUdpServer { for id in ids { self.remove_session(id, false).await; } - while tasks.join_next().await.is_some() {} + let drain = async { while tasks.join_next().await.is_some() {} }; + if time::timeout(Duration::from_secs(2), drain).await.is_err() { + tasks.abort_all(); + while tasks.join_next().await.is_some() {} + } Ok(()) } - async fn handle_session_packet( - &self, - packet_type: PacketType, - id: u64, - sequence: u32, - tag: [u8; 16], - payload: Bytes, - peer: SocketAddr, - ) -> Result<()> { + async fn handle_session_packet(&self, packet: Packet, peer: SocketAddr) -> Result { + let (packet_type, version, flags, id, sequence, tag, payload) = match packet { + Packet::Data(data) => ( + PacketType::Data, + data.version, + data.flags, + data.session_id, + data.sequence, + data.auth_tag, + data.payload, + ), + Packet::Keepalive(packet) | Packet::Close(packet) => ( + packet.packet_type, + packet.version, + packet.flags, + packet.session_id, + packet.sequence, + packet.auth_tag, + Bytes::new(), + ), + _ => { + self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); + return Ok(false); + } + }; let session = match self.sessions.lock().await.get(&id).cloned() { Some(s) => s, None => { self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); - return Ok(()); + return Ok(false); } }; if payload.len() > session.max_datagram { @@ -565,13 +871,13 @@ impl DirectUdpServer { .oversized_datagrams .fetch_add(1, Ordering::Relaxed); self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); - return Ok(()); + return Ok(false); } if !auth::verify_data_auth_tag( &session.key, - protocol::PROTOCOL_VERSION, + version, packet_type, - 0, + flags, id, sequence, &payload, @@ -581,11 +887,11 @@ impl DirectUdpServer { .authentication_failures .fetch_add(1, Ordering::Relaxed); self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); - return Ok(()); + return Ok(false); } if !session.receive_window.lock().await.accept(sequence) { self.stats.datagrams_dropped.fetch_add(1, Ordering::Relaxed); - return Ok(()); + return Ok(false); } *session.peer.lock().await = peer; *session.last_activity.lock().await = Instant::now(); @@ -597,7 +903,7 @@ impl DirectUdpServer { PacketType::Keepalive => {} _ => {} } - Ok(()) + Ok(true) } } diff --git a/src/transport/http3.rs b/src/transport/http3.rs index 7e45053..f52f787 100644 --- a/src/transport/http3.rs +++ b/src/transport/http3.rs @@ -1,21 +1,41 @@ //! HTTP/3 CONNECT-UDP protocol primitives. //! -//! `h3` 0.0.8 exposes the extended-CONNECT and HTTP-Datagram settings, while -//! `h3-quinn` 0.0.10 exposes the actual datagram adapter only behind its -//! `datagram` feature. This module therefore keeps the wire-level -//! CONNECT-UDP/context/capsule logic independent of that optional adapter. -//! Callers can implement [`HttpDatagramIo`] for the h3-datagram handlers once -//! the feature is enabled; application packets are never sent as HTTP body -//! streams. +//! The client and server helpers establish RFC 9220 extended CONNECT with the +//! `connect-udp` protocol, authenticate using bounded RFC 9297 capsules, and +//! carry application packets exclusively in HTTP Datagrams with context ID +//! zero. The pinned h3-quinn adapter supplies the RFC 9297 quarter-stream +//! header; this module enforces the remaining context and size limits. -use std::{fmt, future::Future, pin::Pin, str::FromStr, time::Duration}; +use std::{ + collections::HashMap, + fmt, + future::Future, + net::{SocketAddr, ToSocketAddrs}, + pin::Pin, + str::FromStr, + sync::{Arc, Mutex as StdMutex, Once}, + time::Duration, +}; -use bytes::Bytes; +use bytes::{Buf, Bytes, BytesMut}; +use h3_datagram::datagram_handler::{DatagramReader, DatagramSender, HandleDatagramsExt}; use hmac::{Hmac, Mac}; use rand::{rngs::OsRng, RngCore}; use sha2::Sha256; use subtle::ConstantTimeEq; +use tokio::{sync::Mutex, task::JoinHandle}; use tokio_util::sync::CancellationToken; +use tracing::debug; + +use crate::{ + config::{ClientConfig, ServerConfig}, + protocol::RejectReason, +}; + +use super::{ + admission_error, AdmissionController, PendingHandshake, SessionPermit, TransportConnector, + TransportError, TransportFuture, TransportSession, +}; pub const CONNECT_UDP_CONTEXT_ID: u64 = 0; pub const MAX_QUIC_VARINT: u64 = (1 << 62) - 1; @@ -40,6 +60,8 @@ pub enum Http3Error { VarintOverflow, Truncated, InvalidCapsule, + AuthenticationRejected, + DatagramsUnavailable, ContextMismatch { expected: u64, actual: u64 }, Oversized { length: usize, maximum: usize }, Io(String), @@ -54,6 +76,8 @@ impl fmt::Display for Http3Error { Self::VarintOverflow => write!(f, "QUIC variable integer overflow"), Self::Truncated => write!(f, "truncated HTTP/3 value"), Self::InvalidCapsule => write!(f, "invalid HTTP capsule"), + Self::AuthenticationRejected => write!(f, "CONNECT-UDP authentication rejected"), + Self::DatagramsUnavailable => write!(f, "HTTP/3 datagrams are unavailable"), Self::ContextMismatch { expected, actual } => { write!( f, @@ -279,22 +303,48 @@ pub fn encode_auth_challenge(challenge: &AuthChallenge) -> Result Result { + let (kind, payload) = decode_capsule(capsule)?; + if kind != AUTH_CHALLENGE_CAPSULE || payload.len() != CHALLENGE_LEN { + return Err(Http3Error::InvalidCapsule); + } + let mut bytes = [0u8; CHALLENGE_LEN]; + bytes.copy_from_slice(&payload); + Ok(AuthChallenge { bytes }) +} + pub fn encode_auth_accept() -> Result { encode_capsule(AUTH_ACCEPT_CAPSULE, &[]) } +fn encode_auth_accept_limit(maximum_datagram: usize) -> Result { + let maximum_datagram = u16::try_from(maximum_datagram).map_err(|_| { + Http3Error::InvalidRequest("negotiated datagram maximum exceeds 65535 bytes") + })?; + encode_capsule(AUTH_ACCEPT_CAPSULE, &maximum_datagram.to_be_bytes()) +} + pub fn encode_auth_reject() -> Result { encode_capsule(AUTH_REJECT_CAPSULE, &[]) } pub fn is_auth_accept(capsule: &[u8]) -> Result { let (kind, payload) = decode_capsule(capsule)?; - if !payload.is_empty() { - return Err(Http3Error::InvalidCapsule); - } match kind { - AUTH_ACCEPT_CAPSULE => Ok(true), - AUTH_REJECT_CAPSULE => Ok(false), + AUTH_ACCEPT_CAPSULE if payload.is_empty() || payload.len() == 2 => Ok(true), + AUTH_REJECT_CAPSULE if payload.is_empty() => Ok(false), + _ => Err(Http3Error::InvalidCapsule), + } +} + +fn auth_accept_limit(capsule: &[u8]) -> Result, Http3Error> { + let (kind, payload) = decode_capsule(capsule)?; + match kind { + AUTH_ACCEPT_CAPSULE if payload.is_empty() => Ok(None), + AUTH_ACCEPT_CAPSULE if payload.len() == 2 => { + Ok(Some(u16::from_be_bytes([payload[0], payload[1]]) as usize)) + } + AUTH_REJECT_CAPSULE if payload.is_empty() => Err(Http3Error::AuthenticationRejected), _ => Err(Http3Error::InvalidCapsule), } } @@ -351,9 +401,14 @@ pub fn verify_auth_response( }) } -/// Narrow adapter boundary for the optional h3-datagram implementation. +/// Narrow adapter boundary for the h3-datagram implementation. pub trait HttpDatagramIo: Send + Sync { type Error: fmt::Display + Send + Sync + 'static; + /// Maximum size of the HTTP Datagram payload before its context ID is + /// added. This accounts for the HTTP/3 quarter-stream ID and QUIC limit. + fn maximum_encoded_size(&self) -> Option { + None + } fn send<'a>( &'a self, encoded: Bytes, @@ -372,6 +427,13 @@ pub struct ConnectUdpSession { impl ConnectUdpSession { pub fn new(io: I, maximum_datagram: usize) -> Self { + let context_len = encode_varint(CONNECT_UDP_CONTEXT_ID) + .map(|value| value.len()) + .unwrap_or(1); + let maximum_datagram = io + .maximum_encoded_size() + .map(|wire| wire.saturating_sub(context_len).min(maximum_datagram)) + .unwrap_or(maximum_datagram); Self { io, context_id: CONNECT_UDP_CONTEXT_ID, @@ -395,6 +457,16 @@ impl ConnectUdpSession { return Err(Http3Error::Closed); } let encoded = encode_http_datagram(self.context_id, payload, self.maximum_datagram)?; + if self + .io + .maximum_encoded_size() + .is_some_and(|maximum| encoded.len() > maximum) + { + return Err(Http3Error::Oversized { + length: encoded.len(), + maximum: self.io.maximum_encoded_size().unwrap_or_default(), + }); + } self.io .send(encoded) .await @@ -407,13 +479,16 @@ impl ConnectUdpSession { } let future = self.io.recv(); let encoded = match self.idle_timeout { - Some(timeout) => tokio::time::timeout(timeout, future) - .await - .map_err(|_| Http3Error::IdleTimeout)? - .map_err(|error| Http3Error::Io(error.to_string()))?, - None => future - .await - .map_err(|error| Http3Error::Io(error.to_string()))?, + Some(timeout) => tokio::select! { + _ = self.closed.cancelled() => return Err(Http3Error::Closed), + result = tokio::time::timeout(timeout, future) => result + .map_err(|_| Http3Error::IdleTimeout)? + .map_err(|error| Http3Error::Io(error.to_string()))?, + }, + None => tokio::select! { + _ = self.closed.cancelled() => return Err(Http3Error::Closed), + result = future => result.map_err(|error| Http3Error::Io(error.to_string()))?, + }, }; decode_http_datagram(&encoded, self.context_id, self.maximum_datagram) } @@ -426,3 +501,1092 @@ impl ConnectUdpSession { .map_err(|error| Http3Error::Io(error.to_string())) } } + +const MAX_AUTH_CAPSULE_WIRE_SIZE: usize = 512; +type H3ClientRequestStream = h3::client::RequestStream, Bytes>; +type H3ServerRequestStream = h3::server::RequestStream, Bytes>; +type H3DatagramSender = DatagramSender; +type H3DatagramReader = DatagramReader; +type H3BodyFuture<'a, T> = Pin> + Send + 'a>>; + +trait HttpBodyStream: Send { + fn send_body<'a>(&'a mut self, bytes: Bytes) -> H3BodyFuture<'a, ()>; + fn recv_body<'a>(&'a mut self) -> H3BodyFuture<'a, Option>; + fn finish<'a>(&'a mut self) -> H3BodyFuture<'a, ()>; +} + +impl HttpBodyStream for H3ClientRequestStream { + fn send_body<'a>(&'a mut self, bytes: Bytes) -> H3BodyFuture<'a, ()> { + Box::pin(async move { self.send_data(bytes).await.map_err(|e| e.to_string()) }) + } + + fn recv_body<'a>(&'a mut self) -> H3BodyFuture<'a, Option> { + Box::pin(async move { + let Some(mut data) = self.recv_data().await.map_err(|e| e.to_string())? else { + return Ok(None); + }; + Ok(Some(data.copy_to_bytes(data.remaining()))) + }) + } + + fn finish<'a>(&'a mut self) -> H3BodyFuture<'a, ()> { + Box::pin(async move { self.finish().await.map_err(|e| e.to_string()) }) + } +} + +impl HttpBodyStream for H3ServerRequestStream { + fn send_body<'a>(&'a mut self, bytes: Bytes) -> H3BodyFuture<'a, ()> { + Box::pin(async move { self.send_data(bytes).await.map_err(|e| e.to_string()) }) + } + + fn recv_body<'a>(&'a mut self) -> H3BodyFuture<'a, Option> { + Box::pin(async move { + let Some(mut data) = self.recv_data().await.map_err(|e| e.to_string())? else { + return Ok(None); + }; + Ok(Some(data.copy_to_bytes(data.remaining()))) + }) + } + + fn finish<'a>(&'a mut self) -> H3BodyFuture<'a, ()> { + Box::pin(async move { self.finish().await.map_err(|e| e.to_string()) }) + } +} + +struct CapsuleStream { + stream: S, + pending: BytesMut, +} + +impl CapsuleStream { + fn new(stream: S) -> Self { + Self { + stream, + pending: BytesMut::new(), + } + } + + async fn send_capsule(&mut self, capsule: Bytes) -> Result<(), Http3Error> { + self.stream.send_body(capsule).await.map_err(Http3Error::Io) + } + + async fn recv_capsule(&mut self) -> Result { + loop { + if let Some(frame_len) = capsule_frame_len(&self.pending)? { + if self.pending.len() >= frame_len { + return Ok(self.pending.split_to(frame_len).freeze()); + } + } + if self.pending.len() >= MAX_AUTH_CAPSULE_WIRE_SIZE { + return Err(Http3Error::InvalidCapsule); + } + let Some(chunk) = self.stream.recv_body().await.map_err(Http3Error::Io)? else { + return Err(Http3Error::Truncated); + }; + if self.pending.len().saturating_add(chunk.len()) > MAX_AUTH_CAPSULE_WIRE_SIZE { + return Err(Http3Error::InvalidCapsule); + } + self.pending.extend_from_slice(&chunk); + } + } + + fn has_trailing_data(&self) -> bool { + !self.pending.is_empty() + } +} + +fn capsule_frame_len(buffer: &[u8]) -> Result, Http3Error> { + let (kind, kind_len) = match decode_varint(buffer) { + Ok(value) => value, + Err(Http3Error::Truncated) => return Ok(None), + Err(error) => return Err(error), + }; + let _ = kind; + let (payload_len, length_len) = match decode_varint(&buffer[kind_len..]) { + Ok(value) => value, + Err(Http3Error::Truncated) => return Ok(None), + Err(error) => return Err(error), + }; + let payload_len = usize::try_from(payload_len).map_err(|_| Http3Error::VarintOverflow)?; + let frame_len = kind_len + .checked_add(length_len) + .and_then(|size| size.checked_add(payload_len)) + .ok_or(Http3Error::VarintOverflow)?; + if frame_len > MAX_AUTH_CAPSULE_WIRE_SIZE { + return Err(Http3Error::InvalidCapsule); + } + Ok(Some(frame_len)) +} + +trait SessionControl: Send { + fn finish<'a>(&'a mut self) -> H3BodyFuture<'a, ()>; +} + +impl SessionControl for CapsuleStream { + fn finish<'a>(&'a mut self) -> H3BodyFuture<'a, ()> { + Box::pin(async move { self.stream.finish().await }) + } +} + +struct H3DatagramIo { + sender: Mutex, + reader: Mutex, + stream_id: h3::quic::StreamId, + maximum_encoded_size: usize, + closed: CancellationToken, +} + +impl H3DatagramIo { + fn new( + sender: H3DatagramSender, + reader: H3DatagramReader, + stream_id: h3::quic::StreamId, + maximum_encoded_size: usize, + ) -> Self { + Self { + sender: Mutex::new(sender), + reader: Mutex::new(reader), + stream_id, + maximum_encoded_size, + closed: CancellationToken::new(), + } + } +} + +impl HttpDatagramIo for H3DatagramIo { + type Error = String; + + fn maximum_encoded_size(&self) -> Option { + Some(self.maximum_encoded_size) + } + + fn send<'a>( + &'a self, + encoded: Bytes, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + if self.closed.is_cancelled() { + return Err("session closed".into()); + } + if encoded.len() > self.maximum_encoded_size { + return Err(format!( + "HTTP Datagram payload {} exceeds wire limit {}", + encoded.len(), + self.maximum_encoded_size + )); + } + self.sender + .lock() + .await + .send_datagram(encoded) + .map_err(|e| e.to_string()) + }) + } + + fn recv<'a>(&'a self) -> Pin> + Send + 'a>> { + Box::pin(async move { + loop { + if self.closed.is_cancelled() { + return Err("session closed".into()); + } + let datagram = { + let mut reader = self.reader.lock().await; + tokio::select! { + _ = self.closed.cancelled() => return Err("session closed".into()), + result = reader.read_datagram() => result.map_err(|e| e.to_string())?, + } + }; + if datagram.stream_id() == self.stream_id { + let mut payload = datagram.into_payload(); + return Ok(payload.copy_to_bytes(payload.remaining())); + } + debug!( + received_stream_id = datagram.stream_id().into_inner(), + expected_stream_id = self.stream_id.into_inner(), + "discarding HTTP Datagram for another request stream" + ); + } + }) + } + + fn close<'a>(&'a self) -> Pin> + Send + 'a>> { + Box::pin(async move { + self.closed.cancel(); + Ok(()) + }) + } +} + +/// An established authenticated CONNECT-UDP tunnel. Each HTTP/3 connection +/// currently carries one tunnel so its single datagram reader has one owner. +pub struct Http3ConnectUdpSession { + datagrams: ConnectUdpSession, + control: Mutex>>, + quic_connection: quinn::Connection, + client_id: String, + driver: Mutex>>, + client_sender: Mutex>>, +} + +impl Http3ConnectUdpSession { + pub fn max_datagram_size(&self) -> usize { + self.datagrams.max_datagram_size() + } + + pub fn remote_addr(&self) -> SocketAddr { + self.quic_connection.remote_address() + } + + pub fn rtt(&self) -> Duration { + self.quic_connection.rtt() + } + + pub fn client_id(&self) -> &str { + &self.client_id + } + + pub async fn send_datagram(&self, payload: &[u8]) -> Result<(), Http3Error> { + self.datagrams.send_datagram(payload).await + } + + pub async fn recv_datagram(&self) -> Result { + self.datagrams.recv_datagram().await + } + + pub async fn close(&self, timeout: Duration) -> Result<(), Http3Error> { + self.datagrams.close(timeout).await?; + let finish_result = { + let mut control = self.control.lock().await; + match control.take() { + Some(mut control) => tokio::time::timeout(timeout, control.finish()) + .await + .map_err(|_| Http3Error::Io("CONNECT-UDP close timed out".into()))?, + None => Ok(()), + } + }; + self.quic_connection + .close(0u32.into(), b"CONNECT-UDP closed"); + self.client_sender.lock().await.take(); + if let Some(mut driver) = self.driver.lock().await.take() { + if tokio::time::timeout(timeout, &mut driver).await.is_err() { + // Closing the QUIC connection above is authoritative; abort + // the driver only if it did not observe that close in time. + driver.abort(); + let _ = driver.await; + } + } + finish_result.map_err(Http3Error::Io) + } +} + +/// HTTP/3 client endpoint. `connect_udp` consumes the connection and creates +/// one CONNECT-UDP request with challenge-response authentication. +pub struct Http3ClientConnection { + quic_connection: Option, + h3_connection: Arc>>, + send_request: Option>, + driver: Option>, +} + +impl Http3ClientConnection { + /// Build an HTTP/3 client and start its control-stream driver on an + /// established QUIC connection with QUIC DATAGRAM support. + pub async fn new(quic_connection: quinn::Connection) -> Result { + if quic_connection.max_datagram_size().is_none() { + return Err(Http3Error::DatagramsUnavailable); + } + let mut builder = client_builder(); + let (h3_connection, send_request) = builder + .build::<_, _, Bytes>(h3_quinn::Connection::new(quic_connection.clone())) + .await + .map_err(|e| Http3Error::Io(e.to_string()))?; + let h3_connection = Arc::new(StdMutex::new(h3_connection)); + let driver_connection = h3_connection.clone(); + let driver = tokio::spawn(async move { + let error = std::future::poll_fn(|cx| { + let mut connection = driver_connection + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + connection.poll_close(cx) + }) + .await; + debug!(error = %error, "HTTP/3 client connection driver stopped"); + }); + Ok(Self { + quic_connection: Some(quic_connection), + h3_connection, + send_request: Some(send_request), + driver: Some(driver), + }) + } + + pub async fn connect_udp( + mut self, + authority: &str, + path: &str, + client_id: &str, + secret: &[u8], + maximum_datagram: usize, + handshake_timeout: Duration, + ) -> Result { + validate_connect_udp_request("CONNECT", "connect-udp", path, path)?; + let uri = format!("https://{authority}{path}"); + let mut request = http::Request::builder() + .method(http::Method::CONNECT) + .uri(uri) + .body(()) + .map_err(|e| Http3Error::Io(e.to_string()))?; + request + .extensions_mut() + .insert(h3::ext::Protocol::CONNECT_UDP); + let mut stream = self + .send_request + .as_mut() + .ok_or(Http3Error::Closed)? + .send_request(request) + .await + .map_err(|e| Http3Error::Io(e.to_string()))?; + let stream_id = stream.id(); + let (sender, reader) = { + let connection = self + .h3_connection + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + ( + connection.get_datagram_sender(stream_id), + connection.get_datagram_reader(), + ) + }; + let local_maximum = maximum_application_datagram( + self.quic_connection + .as_ref() + .ok_or(Http3Error::Closed)? + .max_datagram_size() + .ok_or(Http3Error::DatagramsUnavailable)?, + stream_id, + CONNECT_UDP_CONTEXT_ID, + maximum_datagram, + )?; + if local_maximum == 0 { + return Err(Http3Error::DatagramsUnavailable); + } + let response = tokio::time::timeout(handshake_timeout, stream.recv_response()) + .await + .map_err(|_| Http3Error::IdleTimeout)? + .map_err(|e| Http3Error::Io(e.to_string()))?; + if response.status() != http::StatusCode::OK { + return Err(Http3Error::InvalidRequest("server rejected CONNECT-UDP")); + } + let mut control = CapsuleStream::new(stream); + let challenge_capsule = tokio::time::timeout(handshake_timeout, control.recv_capsule()) + .await + .map_err(|_| Http3Error::IdleTimeout)??; + let challenge = decode_auth_challenge(&challenge_capsule)?; + let mut nonce = [0u8; CLIENT_NONCE_LEN]; + OsRng.fill_bytes(&mut nonce); + let response = encode_auth_response(&challenge, client_id, &nonce, secret)?; + tokio::time::timeout(handshake_timeout, control.send_capsule(response)) + .await + .map_err(|_| Http3Error::IdleTimeout)??; + let result_capsule = tokio::time::timeout(handshake_timeout, control.recv_capsule()) + .await + .map_err(|_| Http3Error::IdleTimeout)??; + let remote_maximum = + auth_accept_limit(&result_capsule)?.ok_or(Http3Error::InvalidCapsule)?; + if control.has_trailing_data() || remote_maximum == 0 { + return Err(Http3Error::InvalidCapsule); + } + let maximum_datagram = local_maximum.min(remote_maximum); + let io = H3DatagramIo::new( + sender, + reader, + stream_id, + http_datagram_payload_limit( + self.quic_connection + .as_ref() + .ok_or(Http3Error::Closed)? + .max_datagram_size() + .ok_or(Http3Error::DatagramsUnavailable)?, + stream_id, + )?, + ); + Ok(Http3ConnectUdpSession { + datagrams: ConnectUdpSession::new(io, maximum_datagram), + control: Mutex::new(Some(Box::new(control))), + quic_connection: self.quic_connection.take().ok_or(Http3Error::Closed)?, + client_id: client_id.to_owned(), + driver: Mutex::new(self.driver.take()), + client_sender: Mutex::new(self.send_request.take()), + }) + } +} + +impl Drop for Http3ClientConnection { + fn drop(&mut self) { + if let Some(connection) = self.quic_connection.take() { + connection.close(0u32.into(), b"CONNECT-UDP setup ended"); + } + if let Some(driver) = self.driver.take() { + driver.abort(); + } + } +} + +/// HTTP/3 server endpoint. It accepts and authenticates one CONNECT-UDP +/// request on this QUIC connection before returning an application session. +pub struct Http3ServerConnection { + quic_connection: quinn::Connection, + h3_connection: Option>, +} + +impl Http3ServerConnection { + /// Build an HTTP/3 server connection with extended CONNECT and datagrams + /// enabled. + pub async fn new(quic_connection: quinn::Connection) -> Result { + if quic_connection.max_datagram_size().is_none() { + return Err(Http3Error::DatagramsUnavailable); + } + let builder = server_builder(); + let h3_connection = builder + .build::<_, Bytes>(h3_quinn::Connection::new(quic_connection.clone())) + .await + .map_err(|e| Http3Error::Io(e.to_string()))?; + Ok(Self { + quic_connection, + h3_connection: Some(h3_connection), + }) + } + + pub async fn accept_connect_udp( + mut self, + configured_path: &str, + clients: &std::collections::HashMap>, + maximum_datagram: usize, + handshake_timeout: Duration, + ) -> Result { + let h3_connection = self.h3_connection.as_mut().ok_or(Http3Error::Closed)?; + let resolver = tokio::time::timeout(handshake_timeout, h3_connection.accept()) + .await + .map_err(|_| Http3Error::IdleTimeout)? + .map_err(|e| Http3Error::Io(e.to_string()))? + .ok_or(Http3Error::Closed)?; + let (request, stream) = resolver + .resolve_request() + .await + .map_err(|e| Http3Error::Io(e.to_string()))?; + let protocol = request + .extensions() + .get::() + .map(h3::ext::Protocol::as_str) + .unwrap_or(""); + let path = request + .uri() + .path_and_query() + .map(|value| value.as_str()) + .unwrap_or("/"); + if let Err(error) = + validate_connect_udp_request(request.method().as_str(), protocol, path, configured_path) + { + let mut stream = stream; + let response = http::Response::builder() + .status(http::StatusCode::BAD_REQUEST) + .body(()) + .map_err(|e| Http3Error::Io(e.to_string()))?; + let _ = stream.send_response(response).await; + let _ = stream.finish().await; + return Err(error); + } + let stream_id = stream.send_id(); + let wire_maximum = self + .quic_connection + .max_datagram_size() + .ok_or(Http3Error::DatagramsUnavailable)?; + let maximum_datagram = maximum_application_datagram( + wire_maximum, + stream_id, + CONNECT_UDP_CONTEXT_ID, + maximum_datagram, + )?; + if maximum_datagram == 0 { + return Err(Http3Error::DatagramsUnavailable); + } + let success = http::Response::builder() + .status(http::StatusCode::OK) + .body(()) + .map_err(|e| Http3Error::Io(e.to_string()))?; + let mut stream = stream; + tokio::time::timeout(handshake_timeout, stream.send_response(success)) + .await + .map_err(|_| Http3Error::IdleTimeout)? + .map_err(|e| Http3Error::Io(e.to_string()))?; + let mut control = CapsuleStream::new(stream); + let challenge = AuthChallenge::random(); + let challenge_capsule = encode_auth_challenge(&challenge)?; + tokio::time::timeout(handshake_timeout, control.send_capsule(challenge_capsule)) + .await + .map_err(|_| Http3Error::IdleTimeout)??; + let response_capsule = tokio::time::timeout(handshake_timeout, control.recv_capsule()) + .await + .map_err(|_| Http3Error::IdleTimeout)??; + let client = match verify_auth_response(&challenge, &response_capsule, clients) { + Ok(client) => client, + Err(error) => { + let reject = encode_auth_reject()?; + let _ = tokio::time::timeout(handshake_timeout, control.send_capsule(reject)).await; + let _ = tokio::time::timeout(handshake_timeout, control.stream.finish()).await; + if let Some(connection) = self.h3_connection.take() { + tokio::spawn(drive_server_connection(connection)); + } + return Err( + if matches!( + error, + Http3Error::InvalidRequest("authentication failed" | "unknown client") + ) { + Http3Error::AuthenticationRejected + } else { + error + }, + ); + } + }; + let authenticated_client_id = client.id; + let accept = encode_auth_accept_limit(maximum_datagram)?; + tokio::time::timeout(handshake_timeout, control.send_capsule(accept)) + .await + .map_err(|_| Http3Error::IdleTimeout)??; + if control.has_trailing_data() { + let _ = tokio::time::timeout(handshake_timeout, control.stream.finish()).await; + return Err(Http3Error::InvalidCapsule); + } + + let h3_connection = self.h3_connection.as_mut().ok_or(Http3Error::Closed)?; + let sender = h3_connection.get_datagram_sender(stream_id); + let reader = h3_connection.get_datagram_reader(); + let h3_connection = self.h3_connection.take().ok_or(Http3Error::Closed)?; + let driver = tokio::spawn(async move { + drive_server_connection(h3_connection).await; + }); + let io = H3DatagramIo::new( + sender, + reader, + stream_id, + http_datagram_payload_limit(wire_maximum, stream_id)?, + ); + Ok(Http3ConnectUdpSession { + datagrams: ConnectUdpSession::new(io, maximum_datagram), + control: Mutex::new(Some(Box::new(control))), + quic_connection: self.quic_connection, + client_id: authenticated_client_id, + driver: Mutex::new(Some(driver)), + client_sender: Mutex::new(None), + }) + } +} + +async fn drive_server_connection( + mut connection: h3::server::Connection, +) { + loop { + match connection.accept().await { + Ok(Some(resolver)) => match resolver.resolve_request().await { + Ok((_request, mut stream)) => { + let response = match http::Response::builder() + .status(http::StatusCode::NOT_FOUND) + .body(()) + { + Ok(response) => response, + Err(error) => { + debug!(%error, "failed to construct HTTP/3 rejection response"); + break; + } + }; + if let Err(error) = stream.send_response(response).await { + debug!(%error, "failed to reject second CONNECT-UDP request"); + break; + } + if let Err(error) = stream.finish().await { + debug!(%error, "failed to finish rejected HTTP/3 request"); + break; + } + } + Err(error) => debug!(%error, "invalid additional HTTP/3 request"), + }, + Ok(None) => break, + Err(error) => { + debug!(%error, "HTTP/3 server connection driver stopped"); + break; + } + } + } +} + +fn http_datagram_payload_limit( + wire_maximum: usize, + stream_id: h3::quic::StreamId, +) -> Result { + let quarter_stream_id = stream_id.into_inner() / 4; + let stream_header_len = encode_varint(quarter_stream_id)?.len(); + Ok(wire_maximum.saturating_sub(stream_header_len)) +} + +/// Return the maximum opaque application payload that fits in one HTTP/3 +/// Datagram after the quarter-stream ID and context ID are encoded. +pub fn maximum_application_datagram( + wire_maximum: usize, + stream_id: h3::quic::StreamId, + context_id: u64, + configured_maximum: usize, +) -> Result { + let stream_header_len = encode_varint(stream_id.into_inner() / 4)?.len(); + let context_header_len = encode_varint(context_id)?.len(); + Ok(configured_maximum.min(wire_maximum.saturating_sub(stream_header_len + context_header_len))) +} + +fn install_ring_provider() { + static INSTALL: Once = Once::new(); + INSTALL.call_once(|| { + // Another rustls user may already have installed a process provider. + // HTTP/3 configs below select ring explicitly, so that is harmless. + let _ = rustls::crypto::ring::default_provider().install_default(); + }); +} + +fn h3_io_error(error: impl fmt::Display) -> TransportError { + TransportError::Io(std::io::Error::new( + std::io::ErrorKind::ConnectionAborted, + error.to_string(), + )) +} + +fn map_transport_error(error: Http3Error) -> TransportError { + match error { + Http3Error::AuthenticationRejected => { + TransportError::AuthenticationRejected(RejectReason::AuthenticationFailed) + } + Http3Error::Oversized { maximum, .. } => TransportError::Oversized { maximum }, + Http3Error::DatagramsUnavailable => { + TransportError::Unsupported("HTTP/3 DATAGRAM support was not negotiated".to_owned()) + } + Http3Error::Closed => TransportError::Closed, + Http3Error::IdleTimeout => TransportError::HandshakeTimeout, + other => TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + other.to_string(), + )), + } +} + +/// Runtime adapter for an authenticated HTTP/3 tunnel. The endpoint handle is +/// retained for client sessions because Quinn endpoints own their driver. +pub struct Http3TransportSession { + inner: Http3ConnectUdpSession, + _endpoint: Option, + admission: StdMutex>, +} + +impl Http3TransportSession { + fn new( + inner: Http3ConnectUdpSession, + endpoint: Option, + admission: Option, + ) -> Self { + Self { + inner, + _endpoint: endpoint, + admission: StdMutex::new(admission), + } + } +} + +impl TransportSession for Http3TransportSession { + fn send_datagram<'a>(&'a self, data: Bytes) -> TransportFuture<'a, ()> { + Box::pin(async move { + self.inner + .send_datagram(&data) + .await + .map_err(map_transport_error) + }) + } + + fn recv_datagram<'a>(&'a self) -> TransportFuture<'a, Bytes> { + Box::pin(async move { + self.inner + .recv_datagram() + .await + .map_err(map_transport_error) + }) + } + + fn close<'a>(&'a self) -> TransportFuture<'a, ()> { + Box::pin(async move { + let result = tokio::time::timeout( + Duration::from_secs(3), + self.inner.close(Duration::from_secs(2)), + ) + .await + .map_err(|_| TransportError::HandshakeTimeout) + .and_then(|result| result.map_err(map_transport_error)); + self.admission + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + result + }) + } + + fn max_datagram_size(&self) -> usize { + self.inner.max_datagram_size() + } + + fn client_id(&self) -> Option<&str> { + Some(self.inner.client_id()) + } + + fn remote_addr(&self) -> Option { + Some(self.inner.remote_addr()) + } + + fn rtt(&self) -> Option { + Some(self.inner.rtt()) + } +} + +/// Establishes a normally verified QUIC/TLS connection, then authenticates +/// one CONNECT-UDP tunnel using the configured project credentials. +pub struct Http3ClientConnector { + config: ClientConfig, + client_config: Option, +} + +impl Http3ClientConnector { + pub fn new(config: ClientConfig) -> Self { + Self { + config, + client_config: None, + } + } + + /// Supply a rustls-backed Quinn config, primarily useful for managed trust + /// stores and tests. It must retain certificate and hostname validation. + pub fn with_client_config(config: ClientConfig, client_config: quinn::ClientConfig) -> Self { + Self { + config, + client_config: Some(client_config), + } + } + + async fn connect_session(&self) -> Result, TransportError> { + let host = self.config.client.server.trim(); + let address = (host, self.config.transport.http3.port) + .to_socket_addrs() + .map_err(TransportError::Io)? + .next() + .ok_or_else(|| { + TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "server hostname resolved to no address", + )) + })?; + let secret = self + .config + .client + .secret + .load_bytes("client") + .map_err(|error| { + TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + error.to_string(), + )) + })?; + let endpoint = quinn::Endpoint::client("[::]:0".parse().expect("valid wildcard address")) + .map_err(TransportError::Io)?; + let mut client_config = match self.client_config.clone() { + Some(config) => config, + None => platform_client_config()?, + }; + let mut transport = quinn::TransportConfig::default(); + transport.datagram_send_buffer_size(1024 * 1024); + client_config.transport_config(Arc::new(transport)); + let mut endpoint = endpoint; + endpoint.set_default_client_config(client_config); + let handshake_timeout = + Duration::from_millis(self.config.transport.auto.probe_timeout_ms.max(1)); + let connection = tokio::time::timeout( + handshake_timeout, + endpoint.connect(address, host).map_err(|error| { + TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + error.to_string(), + )) + })?, + ) + .await + .map_err(|_| TransportError::HandshakeTimeout)? + .map_err(h3_io_error)?; + let tunnel = Http3ClientConnection::new(connection) + .await + .map_err(map_transport_error)? + .connect_udp( + &format!("{host}:{}", self.config.transport.http3.port), + &self.config.transport.http3.path, + &self.config.client.id, + &secret, + usize::from(self.config.network.max_datagram_size), + handshake_timeout, + ) + .await + .map_err(map_transport_error)?; + Ok(Box::new(Http3TransportSession::new( + tunnel, + Some(endpoint), + None, + ))) + } +} + +impl TransportConnector for Http3ClientConnector { + fn connect<'a>(&'a self) -> TransportFuture<'a, Box> { + Box::pin(async move { self.connect_session().await }) + } +} + +fn platform_client_config() -> Result { + install_ring_provider(); + let native = rustls_native_certs::load_native_certs(); + let mut roots = rustls::RootCertStore::empty(); + for certificate in native.certs { + roots.add(certificate).map_err(|error| { + TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + error.to_string(), + )) + })?; + } + if roots.is_empty() { + return Err(TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "no system trust roots available", + ))); + } + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let mut tls = rustls::ClientConfig::builder_with_provider(provider) + .with_protocol_versions(&[&rustls::version::TLS13]) + .map_err(|error| { + TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + error.to_string(), + )) + })? + .with_root_certificates(roots) + .with_no_client_auth(); + tls.alpn_protocols = vec![b"h3".to_vec()]; + let crypto = quinn::crypto::rustls::QuicClientConfig::try_from(tls).map_err(|error| { + TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + error.to_string(), + )) + })?; + Ok(quinn::ClientConfig::new(Arc::new(crypto))) +} + +/// HTTP/3 QUIC listener. Authentication completes before `accept_session` +/// returns, which is the server runtime's gate for backend socket creation. +pub struct Http3Server { + endpoint: quinn::Endpoint, + clients: Arc>>, + path: String, + maximum_datagram: usize, + handshake_timeout: Duration, + admission: Arc, +} + +pub struct PendingHttp3Handshake { + incoming: quinn::Incoming, + admission: PendingHandshake, +} + +impl Http3Server { + pub async fn bind(config: &ServerConfig) -> Result { + Self::bind_with_admission(config, AdmissionController::new(&config.limits)).await + } + + pub async fn bind_with_admission( + config: &ServerConfig, + admission: Arc, + ) -> Result { + install_ring_provider(); + let address = resolve_http3_listen(&config.server.listen, config.transports.http3.port)?; + let certificate = config.tls.certificate.as_ref().ok_or_else(|| { + TransportError::Unsupported( + "HTTP/3 requires tls.certificate and tls.private_key".into(), + ) + })?; + let private_key = config.tls.private_key.as_ref().ok_or_else(|| { + TransportError::Unsupported( + "HTTP/3 requires tls.certificate and tls.private_key".into(), + ) + })?; + let (certificates, key) = super::quic::load_tls_identity(certificate, private_key)?; + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let mut tls = rustls::ServerConfig::builder_with_provider(provider) + .with_protocol_versions(&[&rustls::version::TLS13]) + .map_err(|error| { + TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + error.to_string(), + )) + })? + .with_no_client_auth() + .with_single_cert(certificates, key) + .map_err(|error| { + TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + error.to_string(), + )) + })?; + tls.alpn_protocols = vec![b"h3".to_vec()]; + let crypto = quinn::crypto::rustls::QuicServerConfig::try_from(tls).map_err(|error| { + TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + error.to_string(), + )) + })?; + let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(crypto)); + let mut transport = quinn::TransportConfig::default(); + transport.datagram_send_buffer_size(1024 * 1024); + server_config.transport_config(Arc::new(transport)); + let endpoint = + quinn::Endpoint::server(server_config, address).map_err(TransportError::Io)?; + let mut clients = HashMap::with_capacity(config.auth.clients.len()); + for client in &config.auth.clients { + let secret = client + .secret + .load_bytes(&format!("auth client {}", client.id)) + .map_err(|error| { + TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + error.to_string(), + )) + })?; + clients.insert(client.id.clone(), secret); + } + Ok(Self { + endpoint, + clients: Arc::new(clients), + path: config.transports.http3.path.clone(), + maximum_datagram: DEFAULT_MAX_DATAGRAM, + handshake_timeout: Duration::from_millis(config.limits.handshake_timeout_ms.max(1)), + admission, + }) + } + + pub fn from_endpoint( + endpoint: quinn::Endpoint, + path: impl Into, + clients: HashMap>, + maximum_datagram: usize, + handshake_timeout: Duration, + ) -> Self { + Self::from_endpoint_with_admission( + endpoint, + path, + clients, + maximum_datagram, + handshake_timeout, + AdmissionController::new(&crate::config::LimitsConfig::default()), + ) + } + + pub fn from_endpoint_with_admission( + endpoint: quinn::Endpoint, + path: impl Into, + clients: HashMap>, + maximum_datagram: usize, + handshake_timeout: Duration, + admission: Arc, + ) -> Self { + Self { + endpoint, + clients: Arc::new(clients), + path: path.into(), + maximum_datagram: maximum_datagram.max(1), + handshake_timeout, + admission, + } + } + + pub fn local_addr(&self) -> Result { + self.endpoint.local_addr().map_err(TransportError::Io) + } + + pub async fn accept_pending(&self) -> Result { + let incoming = self.endpoint.accept().await.ok_or(TransportError::Closed)?; + let pending = self + .admission + .begin(incoming.remote_address()) + .map_err(admission_error)?; + Ok(PendingHttp3Handshake { + incoming, + admission: pending, + }) + } + + pub async fn authenticate_pending( + &self, + pending: PendingHttp3Handshake, + ) -> Result { + tokio::time::timeout( + self.handshake_timeout, + self.finish_handshake(pending.incoming, pending.admission), + ) + .await + .map_err(|_| TransportError::HandshakeTimeout)? + } + + pub async fn accept_session(&self) -> Result { + let pending = self.accept_pending().await?; + self.authenticate_pending(pending).await + } + + async fn finish_handshake( + &self, + incoming: quinn::Incoming, + pending: PendingHandshake, + ) -> Result { + let connection = tokio::time::timeout(self.handshake_timeout, incoming) + .await + .map_err(|_| TransportError::HandshakeTimeout)? + .map_err(h3_io_error)?; + let session = Http3ServerConnection::new(connection) + .await + .map_err(map_transport_error)? + .accept_connect_udp( + &self.path, + &self.clients, + self.maximum_datagram, + self.handshake_timeout, + ) + .await + .map_err(map_transport_error)?; + let admission = pending.promote(); + Ok(Http3TransportSession::new(session, None, Some(admission))) + } + + pub fn close(&self) { + self.endpoint.close(0u32.into(), b"server stopped"); + } +} + +fn resolve_http3_listen(host: &str, port: u16) -> Result { + let address = if host.contains(':') && !host.starts_with('[') { + format!("[{host}]:{port}") + } else { + format!("{host}:{port}") + }; + address.parse::().map_err(|error| { + TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("invalid HTTP/3 listen address: {error}"), + )) + }) +} diff --git a/src/transport/mod.rs b/src/transport/mod.rs index ff8edc5..e431c35 100644 --- a/src/transport/mod.rs +++ b/src/transport/mod.rs @@ -1,7 +1,13 @@ //! Common transport boundary and the Milestone 1 direct UDP transport. use std::{ - collections::HashMap, future::Future, net::SocketAddr, pin::Pin, sync::Arc, time::Duration, + collections::{HashMap, VecDeque}, + future::Future, + io, + net::{IpAddr, SocketAddr}, + pin::Pin, + sync::{Arc, Mutex as StdMutex}, + time::{Duration, Instant}, }; use bytes::Bytes; @@ -9,7 +15,7 @@ use thiserror::Error; use tokio::{ io::{split, AsyncRead, AsyncWrite}, net::{TcpListener, TcpStream}, - sync::Mutex, + sync::{mpsc, oneshot, Mutex}, }; pub mod direct_udp; @@ -18,6 +24,7 @@ pub mod quic; pub mod tls_tcp; pub use direct_udp::{DirectUdpConnector, DirectUdpServer, DirectUdpSession}; +pub use http3::{Http3ClientConnector, Http3Server, Http3TransportSession}; pub type TransportFuture<'a, T> = Pin> + Send + 'a>>; @@ -38,16 +45,258 @@ pub enum TransportError { Closed, #[error("transport handshake timed out")] HandshakeTimeout, + #[error("transport server is busy")] + Busy, + #[error("transport authentication attempts are rate limited")] + RateLimited, #[error("transport is not available: {0}")] Unsupported(String), } +const MAX_TRACKED_SOURCE_IPS: usize = 4096; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AdmissionError { + Busy, + RateLimited, +} + +#[derive(Clone, Copy)] +struct AdmissionLimits { + max_sessions: usize, + max_sessions_per_ip: usize, + max_pending_handshakes: usize, + max_auth_attempts_per_ip: usize, + auth_attempt_window: Duration, +} + +#[derive(Default)] +struct AdmissionState { + active: usize, + pending: usize, + active_by_ip: HashMap, + pending_by_ip: HashMap, + attempts_by_ip: HashMap, +} + +#[derive(Default)] +struct AttemptWindow { + attempts: VecDeque, + last_seen: Option, +} + +/// Shared admission controller for encrypted server listeners. +pub struct AdmissionController { + limits: AdmissionLimits, + state: StdMutex, +} + +impl AdmissionController { + pub fn new(config: &crate::config::LimitsConfig) -> Arc { + Arc::new(Self { + limits: AdmissionLimits { + max_sessions: config.max_sessions, + max_sessions_per_ip: config.max_sessions_per_ip, + max_pending_handshakes: config.max_pending_handshakes, + max_auth_attempts_per_ip: config.max_auth_attempts_per_ip, + auth_attempt_window: Duration::from_secs(config.auth_attempt_window_secs.max(1)), + }, + state: StdMutex::new(AdmissionState::default()), + }) + } + + pub fn begin(self: &Arc, remote: SocketAddr) -> Result { + let ip = match remote.ip() { + IpAddr::V6(address) => address + .to_ipv4_mapped() + .map(IpAddr::V4) + .unwrap_or(IpAddr::V6(address)), + address => address, + }; + let now = Instant::now(); + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.attempts_by_ip.retain(|_, window| { + window.last_seen.is_some_and(|last_seen| { + now.duration_since(last_seen) < self.limits.auth_attempt_window + }) + }); + + if state.pending >= self.limits.max_pending_handshakes + || state.active + state.pending >= self.limits.max_sessions + || state.active_by_ip.get(&ip).copied().unwrap_or(0) + + state.pending_by_ip.get(&ip).copied().unwrap_or(0) + >= self.limits.max_sessions_per_ip + { + return Err(AdmissionError::Busy); + } + if !state.attempts_by_ip.contains_key(&ip) + && state.attempts_by_ip.len() >= MAX_TRACKED_SOURCE_IPS + { + return Err(AdmissionError::Busy); + } + let window = state.attempts_by_ip.entry(ip).or_default(); + while window + .attempts + .front() + .is_some_and(|attempt| now.duration_since(*attempt) >= self.limits.auth_attempt_window) + { + window.attempts.pop_front(); + } + if window.attempts.len() >= self.limits.max_auth_attempts_per_ip { + window.last_seen = Some(now); + return Err(AdmissionError::RateLimited); + } + window.attempts.push_back(now); + window.last_seen = Some(now); + state.pending += 1; + *state.pending_by_ip.entry(ip).or_default() += 1; + Ok(PendingHandshake { + controller: self.clone(), + ip, + pending: true, + }) + } + + pub fn active_sessions(&self) -> usize { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .active + } + + pub fn pending_handshakes(&self) -> usize { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .pending + } +} + +pub struct PendingHandshake { + controller: Arc, + ip: IpAddr, + pending: bool, +} + +impl PendingHandshake { + pub fn promote(mut self) -> SessionPermit { + let mut state = self + .controller + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + decrement_pending(&mut state, self.ip); + state.active += 1; + *state.active_by_ip.entry(self.ip).or_default() += 1; + self.pending = false; + SessionPermit { + controller: self.controller.clone(), + ip: self.ip, + active: true, + } + } +} + +impl Drop for PendingHandshake { + fn drop(&mut self) { + if self.pending { + let mut state = self + .controller + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + decrement_pending(&mut state, self.ip); + } + } +} + +pub struct SessionPermit { + controller: Arc, + ip: IpAddr, + active: bool, +} + +impl Drop for SessionPermit { + fn drop(&mut self) { + if self.active { + let mut state = self + .controller + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + decrement_active(&mut state, self.ip); + self.active = false; + } + } +} + +fn decrement_pending(state: &mut AdmissionState, ip: IpAddr) { + state.pending = state.pending.saturating_sub(1); + decrement_ip(&mut state.pending_by_ip, ip); +} + +fn decrement_active(state: &mut AdmissionState, ip: IpAddr) { + state.active = state.active.saturating_sub(1); + decrement_ip(&mut state.active_by_ip, ip); +} + +fn decrement_ip(by_ip: &mut HashMap, ip: IpAddr) { + if let Some(count) = by_ip.get_mut(&ip) { + *count = count.saturating_sub(1); + if *count == 0 { + by_ip.remove(&ip); + } + } +} + +pub fn admission_error(error: AdmissionError) -> TransportError { + match error { + AdmissionError::Busy => TransportError::Busy, + AdmissionError::RateLimited => TransportError::RateLimited, + } +} + /// Logical datagram session shared by all future carrier implementations. pub trait TransportSession: Send + Sync { fn send_datagram<'a>(&'a self, data: Bytes) -> TransportFuture<'a, ()>; fn recv_datagram<'a>(&'a self) -> TransportFuture<'a, Bytes>; fn close<'a>(&'a self) -> TransportFuture<'a, ()>; fn max_datagram_size(&self) -> usize; + + /// Authenticated client identity, when the carrier exposes it to the + /// server runtime. Client-side sessions may also return their configured + /// identity for diagnostics. + fn client_id(&self) -> Option<&str> { + None + } + + /// Send an authenticated transport control keepalive, if supported. + fn send_keepalive<'a>(&'a self) -> TransportFuture<'a, ()> { + Box::pin(async { + Err(TransportError::Unsupported( + "authenticated keepalive is not supported by this transport".into(), + )) + }) + } + + /// Authenticated application-control keepalive cadence. Native transport + /// keepalives should return `None` so the client sends no redundant frame. + fn authenticated_keepalive_interval(&self) -> Option { + None + } + + /// The currently resolved peer endpoint, when the transport exposes it. + fn remote_addr(&self) -> Option { + None + } + + /// Most recent transport RTT measurement, if available. + fn rtt(&self) -> Option { + None + } } /// Connection establishment is separate from an established session. @@ -61,33 +310,141 @@ trait AsyncIo: AsyncRead + AsyncWrite + Send + Unpin {} impl AsyncIo for T {} type BoxIo = Box; +fn tls_auth_error(error: tls_tcp::AuthError) -> TransportError { + match error { + tls_tcp::AuthError::Rejected | tls_tcp::AuthError::InvalidCredentials => { + TransportError::AuthenticationRejected( + crate::protocol::RejectReason::AuthenticationFailed, + ) + } + tls_tcp::AuthError::UnknownClient => { + TransportError::AuthenticationRejected(crate::protocol::RejectReason::UnknownClient) + } + tls_tcp::AuthError::Frame(tls_tcp::FrameError::Io(message)) + if message == "authentication timeout" => + { + TransportError::HandshakeTimeout + } + tls_tcp::AuthError::Frame(tls_tcp::FrameError::Io(message)) => { + TransportError::Io(io::Error::new(io::ErrorKind::ConnectionAborted, message)) + } + tls_tcp::AuthError::Frame(error) => TransportError::Io(io::Error::new( + io::ErrorKind::InvalidData, + error.to_string(), + )), + tls_tcp::AuthError::InvalidMessage => TransportError::Io(io::Error::new( + io::ErrorKind::InvalidData, + "invalid TLS/TCP authentication message", + )), + } +} + struct TlsTcpSession { reader: Mutex>>, - writer: Mutex>>, + writer: mpsc::Sender, max_datagram: usize, closed: tokio_util::sync::CancellationToken, + keepalive_interval: Option, + remote_addr: Option, + client_id: Option, + admission: StdMutex>, +} + +enum TlsWriterCommand { + Datagram(Bytes, oneshot::Sender>), + Keepalive(oneshot::Sender>), + Close(oneshot::Sender>), +} + +struct TlsSessionOptions { + max_datagram: usize, + idle_timeout: Duration, + keepalive_interval: Option, + remote_addr: Option, + client_id: Option, + admission: Option, } impl TlsTcpSession { - fn from_halves( - reader: TlsRead, - writer: TlsWrite, - max_datagram: usize, - idle_timeout: Duration, - ) -> Self { + fn from_halves(reader: TlsRead, writer: TlsWrite, options: TlsSessionOptions) -> Self { + let TlsSessionOptions { + max_datagram, + idle_timeout, + keepalive_interval, + remote_addr, + client_id, + admission, + } = options; + // A dedicated writer task owns each framed stream write to completion. + // Dropping a caller while it awaits the reply cannot leave a partial + // frame on the TCP stream. + let (writer_tx, mut writer_rx) = mpsc::channel(16); + tokio::spawn(async move { + let mut writer = tls_tcp::FramedWriter::new(writer, max_datagram); + while let Some(command) = writer_rx.recv().await { + match command { + TlsWriterCommand::Datagram(data, reply) => { + let result = writer.send_datagram(&data).await; + let failed = result.is_err(); + let _ = reply.send(result); + if failed { + break; + } + } + TlsWriterCommand::Keepalive(reply) => { + let result = writer.send_keepalive().await; + let failed = result.is_err(); + let _ = reply.send(result); + if failed { + break; + } + } + TlsWriterCommand::Close(reply) => { + let result = match writer.send_close().await { + Ok(()) => writer.shutdown(Duration::from_secs(2)).await, + Err(error) => Err(error), + }; + let _ = reply.send(result); + break; + } + } + } + }); Self { reader: Mutex::new(Some(tls_tcp::FramedReader::new( reader, max_datagram, Some(idle_timeout), ))), - writer: Mutex::new(Some(tls_tcp::FramedWriter::new(writer, max_datagram))), + writer: writer_tx, max_datagram, closed: tokio_util::sync::CancellationToken::new(), + keepalive_interval, + remote_addr, + client_id, + admission: StdMutex::new(admission), } } } +fn tls_session_error(error: tls_tcp::SessionError) -> TransportError { + match error { + tls_tcp::SessionError::Closed | tls_tcp::SessionError::IdleTimeout => { + TransportError::Closed + } + tls_tcp::SessionError::Frame(tls_tcp::FrameError::Oversized { maximum, .. }) => { + TransportError::Oversized { maximum } + } + tls_tcp::SessionError::Frame(error) => { + TransportError::Io(std::io::Error::other(error.to_string())) + } + other => TransportError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + other.to_string(), + )), + } +} + impl TransportSession for TlsTcpSession { fn send_datagram<'a>(&'a self, data: Bytes) -> TransportFuture<'a, ()> { Box::pin(async move { @@ -96,24 +453,18 @@ impl TransportSession for TlsTcpSession { maximum: self.max_datagram, }); } - let mut guard = self.writer.lock().await; - let Some(writer) = guard.as_mut() else { + if self.closed.is_cancelled() { return Err(TransportError::Closed); - }; - writer - .send_datagram(&data) + } + let (reply, result) = oneshot::channel(); + self.writer + .send(TlsWriterCommand::Datagram(data, reply)) .await - .map_err(|error| match error { - tls_tcp::SessionError::Frame(tls_tcp::FrameError::Oversized { - maximum, - .. - }) => TransportError::Oversized { maximum }, - tls_tcp::SessionError::Closed => TransportError::Closed, - other => TransportError::Io(std::io::Error::new( - std::io::ErrorKind::Other, - other.to_string(), - )), - }) + .map_err(|_| TransportError::Closed)?; + result + .await + .map_err(|_| TransportError::Closed)? + .map_err(tls_session_error) }) } fn recv_datagram<'a>(&'a self) -> TransportFuture<'a, Bytes> { @@ -133,10 +484,7 @@ impl TransportSession for TlsTcpSession { .. }) => TransportError::Oversized { maximum }, tls_tcp::SessionError::IdleTimeout => TransportError::Closed, - other => TransportError::Io(std::io::Error::new( - std::io::ErrorKind::Other, - other.to_string(), - )), + other => TransportError::Io(std::io::Error::other(other.to_string())), }) }) } @@ -146,25 +494,64 @@ impl TransportSession for TlsTcpSession { return Ok(()); } self.closed.cancel(); - let writer = self.writer.lock().await.take(); - if let Some(mut writer) = writer { - let _ = writer.send_close().await; - writer - .shutdown(Duration::from_secs(2)) + self.admission + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let (reply, result) = oneshot::channel(); + tokio::time::timeout(Duration::from_secs(2), async { + if self + .writer + .send(TlsWriterCommand::Close(reply)) .await - .map_err(|error| { - TransportError::Io(std::io::Error::new( - std::io::ErrorKind::Other, - error.to_string(), - )) - })?; - } - Ok(()) + .is_err() + { + return Ok(()); + } + let result = result.await.map_err(|_| TransportError::Closed)?; + result.map_err(tls_session_error) + }) + .await + .map_err(|_| { + TransportError::Io(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "TLS/TCP close timed out", + )) + })? }) } fn max_datagram_size(&self) -> usize { self.max_datagram } + + fn client_id(&self) -> Option<&str> { + self.client_id.as_deref() + } + + fn send_keepalive<'a>(&'a self) -> TransportFuture<'a, ()> { + Box::pin(async move { + if self.closed.is_cancelled() { + return Err(TransportError::Closed); + } + let (reply, result) = oneshot::channel(); + self.writer + .send(TlsWriterCommand::Keepalive(reply)) + .await + .map_err(|_| TransportError::Closed)?; + result + .await + .map_err(|_| TransportError::Closed)? + .map_err(tls_session_error) + }) + } + + fn authenticated_keepalive_interval(&self) -> Option { + self.keepalive_interval + } + + fn remote_addr(&self) -> Option { + self.remote_addr + } } pub struct TlsTcpConnector { @@ -190,6 +577,7 @@ impl TlsTcpConnector { async fn connect_session(&self) -> Result, TransportError> { let host = self.config.client.server.trim(); let stream = TcpStream::connect((host, self.config.transport.tls_tcp.port)).await?; + let remote_addr = stream.peer_addr().ok(); let connector_config = match &self.client_config { Some(config) => config.clone(), None => { @@ -248,13 +636,21 @@ impl TlsTcpConnector { Duration::from_millis(self.config.transport.auto.probe_timeout_ms.max(1)), ) .await - .map_err(|e| { - TransportError::Io(std::io::Error::new( - std::io::ErrorKind::PermissionDenied, - e.to_string(), - )) - })?; - let session = TlsTcpSession::from_halves(reader, writer, max, Duration::from_secs(120)); + .map_err(tls_auth_error)?; + let keepalive_secs = self.config.transport.tls_tcp.keepalive_secs; + let keepalive_interval = (keepalive_secs > 0).then(|| Duration::from_secs(keepalive_secs)); + let session = TlsTcpSession::from_halves( + reader, + writer, + TlsSessionOptions { + max_datagram: max, + idle_timeout: Duration::from_secs(120), + keepalive_interval, + remote_addr, + client_id: Some(self.config.client.id.clone()), + admission: None, + }, + ); Ok(Box::new(session)) } } @@ -269,9 +665,25 @@ pub struct TlsTcpServer { tls: Arc, clients: Arc>>, max_datagram: usize, + handshake_timeout: Duration, + admission: Arc, } + +pub struct PendingTlsHandshake { + stream: TcpStream, + remote_addr: SocketAddr, + admission: PendingHandshake, +} + impl TlsTcpServer { pub async fn bind(config: &crate::config::ServerConfig) -> Result { + Self::bind_with_admission(config, AdmissionController::new(&config.limits)).await + } + + pub async fn bind_with_admission( + config: &crate::config::ServerConfig, + admission: Arc, + ) -> Result { let listen = if config.server.listen.parse::().is_ok() { config.server.listen.clone() } else { @@ -314,13 +726,51 @@ impl TlsTcpServer { tls, clients: Arc::new(clients), max_datagram: 1200, + handshake_timeout: Duration::from_millis(config.limits.handshake_timeout_ms.max(1)), + admission, }) } pub fn local_addr(&self) -> Result { Ok(self.listener.local_addr()?) } + pub async fn accept_pending(&self) -> Result { + let (stream, remote_addr) = self.listener.accept().await?; + let pending = self.admission.begin(remote_addr).map_err(admission_error)?; + Ok(PendingTlsHandshake { + stream, + remote_addr, + admission: pending, + }) + } + + pub async fn authenticate_pending( + &self, + pending: PendingTlsHandshake, + ) -> Result, TransportError> { + let PendingTlsHandshake { + stream, + remote_addr, + admission, + } = pending; + tokio::time::timeout( + self.handshake_timeout, + self.authenticate_peer(stream, remote_addr, admission), + ) + .await + .map_err(|_| TransportError::HandshakeTimeout)? + } + pub async fn accept_session(&self) -> Result, TransportError> { - let (stream, _) = self.listener.accept().await?; + let pending = self.accept_pending().await?; + self.authenticate_pending(pending).await + } + + async fn authenticate_peer( + &self, + stream: TcpStream, + remote_addr: SocketAddr, + pending: PendingHandshake, + ) -> Result, TransportError> { let acceptor = tokio_rustls::TlsAcceptor::from(self.tls.clone()); let tls = acceptor.accept(stream).await.map_err(|e| { TransportError::Io(std::io::Error::new( @@ -329,25 +779,90 @@ impl TlsTcpServer { )) })?; let (mut reader, mut writer) = split(Box::new(tls) as BoxIo); - tls_tcp::server_authenticate( + let authenticated = tls_tcp::server_authenticate( &mut reader, &mut writer, &self.clients, self.max_datagram.saturating_add(512), - Duration::from_secs(3), + self.handshake_timeout, ) .await - .map_err(|e| { - TransportError::Io(std::io::Error::new( - std::io::ErrorKind::PermissionDenied, - e.to_string(), - )) - })?; + .map_err(tls_auth_error)?; + let admission = pending.promote(); Ok(Box::new(TlsTcpSession::from_halves( reader, writer, - self.max_datagram, - Duration::from_secs(120), + TlsSessionOptions { + max_datagram: self.max_datagram, + idle_timeout: Duration::from_secs(120), + keepalive_interval: None, + remote_addr: Some(remote_addr), + client_id: Some(authenticated.id), + admission: Some(admission), + }, ))) } } + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::{duplex, AsyncReadExt}; + + #[tokio::test] + async fn cancelling_keepalive_wait_does_not_cancel_partial_frame_write() { + let (client, peer) = duplex(1); + let (client_reader, client_writer) = split(Box::new(client) as BoxIo); + let (mut peer_reader, _peer_writer) = split(Box::new(peer) as BoxIo); + let session = Arc::new(TlsTcpSession::from_halves( + client_reader, + client_writer, + TlsSessionOptions { + max_datagram: 1200, + idle_timeout: Duration::from_secs(5), + keepalive_interval: Some(Duration::from_secs(1)), + remote_addr: None, + client_id: None, + admission: None, + }, + )); + + let send_session = session.clone(); + let send = tokio::spawn(async move { send_session.send_keepalive().await }); + let mut first_header_byte = [0; 1]; + tokio::time::timeout( + Duration::from_secs(1), + peer_reader.read_exact(&mut first_header_byte), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(first_header_byte, [0]); + + // The writer is now blocked partway through its frame. Cancelling the + // awaiting caller must not cancel the dedicated stream writer. + send.abort(); + let _ = send.await; + let mut rest = [0; 2]; + tokio::time::timeout(Duration::from_secs(1), peer_reader.read_exact(&mut rest)) + .await + .unwrap() + .unwrap(); + assert_eq!(rest, [1, 0x02]); + + let send_session = session.clone(); + let data_send = + tokio::spawn( + async move { send_session.send_datagram(Bytes::from_static(b"ok")).await }, + ); + let next_frame = tokio::time::timeout( + Duration::from_secs(1), + tls_tcp::read_frame(&mut peer_reader, 1201), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(next_frame, [0x01, b'o', b'k']); + data_send.await.unwrap().unwrap(); + } +} diff --git a/src/transport/quic.rs b/src/transport/quic.rs index 8238c10..672ab94 100644 --- a/src/transport/quic.rs +++ b/src/transport/quic.rs @@ -30,7 +30,10 @@ use crate::{ protocol::{auth, RejectReason, PROTOCOL_VERSION}, }; -use super::{TransportConnector, TransportError, TransportFuture, TransportSession}; +use super::{ + admission_error, AdmissionController, PendingHandshake, SessionPermit, TransportConnector, + TransportError, TransportFuture, TransportSession, +}; const MAGIC: &[u8; 4] = b"WGCQ"; const CONTROL_VERSION: u8 = PROTOCOL_VERSION; @@ -44,7 +47,7 @@ const MAX_CLIENT_ID: usize = 255; const DEFAULT_MAX_DATAGRAM: usize = 1200; fn io_other(message: impl Into) -> io::Error { - io::Error::new(io::ErrorKind::Other, message.into()) + io::Error::other(message.into()) } fn io_invalid(message: impl Into) -> io::Error { @@ -251,16 +254,26 @@ pub struct QuicDatagramSession { connection: Connection, endpoint: Endpoint, max_datagram: usize, + client_id: Option, closed: CancellationToken, + admission: std::sync::Mutex>, } impl QuicDatagramSession { - fn new(connection: Connection, endpoint: Endpoint, max_datagram: usize) -> Self { + fn new( + connection: Connection, + endpoint: Endpoint, + max_datagram: usize, + client_id: Option, + admission: Option, + ) -> Self { Self { connection, endpoint, max_datagram, + client_id, closed: CancellationToken::new(), + admission: std::sync::Mutex::new(admission), } } @@ -320,6 +333,10 @@ impl TransportSession for QuicDatagramSession { if !self.closed.is_cancelled() { self.closed.cancel(); self.connection.close(0u32.into(), b"closed"); + self.admission + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); // Dropping the endpoint handle is intentionally deferred to // session drop; this keeps the connection driver alive while // the close frame is emitted. @@ -332,6 +349,18 @@ impl TransportSession for QuicDatagramSession { fn max_datagram_size(&self) -> usize { self.max_datagram } + + fn client_id(&self) -> Option<&str> { + self.client_id.as_deref() + } + + fn remote_addr(&self) -> Option { + Some(self.connection.remote_address()) + } + + fn rtt(&self) -> Option { + Some(self.connection.rtt()) + } } /// Client-side QUIC connector. @@ -407,7 +436,13 @@ impl QuicClientConnector { ))); } info!(remote_addr=%addr, max_datagram, "QUIC session authenticated"); - Ok(QuicDatagramSession::new(connection, endpoint, max_datagram)) + Ok(QuicDatagramSession::new( + connection, + endpoint, + max_datagram, + Some(self.config.client.id.clone()), + None, + )) } pub async fn connect(&self) -> Result, TransportError> { @@ -429,6 +464,12 @@ pub struct QuicServer { clients: Arc>>, max_datagram: usize, handshake_timeout: Duration, + admission: Arc, +} + +pub struct PendingQuicHandshake { + incoming: quinn::Incoming, + admission: PendingHandshake, } /// Alias matching the listener terminology used in the specification. @@ -436,7 +477,16 @@ pub type QuicListener = QuicServer; impl QuicServer { pub async fn bind(config: &ServerConfig) -> Result { + Self::bind_with_admission(config, AdmissionController::new(&config.limits)).await + } + + pub async fn bind_with_admission( + config: &ServerConfig, + admission: Arc, + ) -> Result { let addr = resolve_listen(&config.server.listen, config.transports.quic.port)?; + // ServerQuicConfig currently has no keepalive setting; keep the + // listener's idle-path probe at the documented 15-second default. let tls = server_config(config, 15); let endpoint = Endpoint::server(tls?, addr).map_err(TransportError::Io)?; let mut clients = HashMap::new(); @@ -452,6 +502,7 @@ impl QuicServer { clients: Arc::new(clients), max_datagram: DEFAULT_MAX_DATAGRAM, handshake_timeout: Duration::from_millis(config.limits.handshake_timeout_ms.max(1)), + admission, }) } @@ -462,12 +513,29 @@ impl QuicServer { clients: HashMap>, max_datagram: usize, handshake_timeout: Duration, + ) -> Self { + Self::from_endpoint_with_admission( + endpoint, + clients, + max_datagram, + handshake_timeout, + AdmissionController::new(&crate::config::LimitsConfig::default()), + ) + } + + pub fn from_endpoint_with_admission( + endpoint: Endpoint, + clients: HashMap>, + max_datagram: usize, + handshake_timeout: Duration, + admission: Arc, ) -> Self { Self { endpoint, clients: Arc::new(clients), max_datagram: max_datagram.max(1), handshake_timeout, + admission, } } @@ -479,8 +547,40 @@ impl QuicServer { &self.endpoint } - pub async fn accept_session(&self) -> Result { + pub async fn accept_pending(&self) -> Result { let incoming = self.endpoint.accept().await.ok_or(TransportError::Closed)?; + let pending = self + .admission + .begin(incoming.remote_address()) + .map_err(admission_error)?; + Ok(PendingQuicHandshake { + incoming, + admission: pending, + }) + } + + pub async fn authenticate_pending( + &self, + pending: PendingQuicHandshake, + ) -> Result { + timeout( + self.handshake_timeout, + self.finish_handshake(pending.incoming, pending.admission), + ) + .await + .map_err(|_| TransportError::HandshakeTimeout)? + } + + pub async fn accept_session(&self) -> Result { + let pending = self.accept_pending().await?; + self.authenticate_pending(pending).await + } + + async fn finish_handshake( + &self, + incoming: quinn::Incoming, + pending: PendingHandshake, + ) -> Result { let connection = timeout(self.handshake_timeout, incoming) .await .map_err(|_| TransportError::HandshakeTimeout)? @@ -533,14 +633,17 @@ impl QuicServer { ))); } let id_bytes = &response[8..8 + id_len]; - let id = std::str::from_utf8(id_bytes).map_err(|_| { - TransportError::AuthenticationRejected(RejectReason::AuthenticationFailed) - })?; + let id = match std::str::from_utf8(id_bytes) { + Ok(id) => id, + Err(_) => { + reject_authentication(&mut send, self.handshake_timeout).await; + return Err(TransportError::AuthenticationRejected( + RejectReason::AuthenticationFailed, + )); + } + }; let Some(secret) = self.clients.get(id) else { - let _ = send_result(&mut send, 1, 0, self.handshake_timeout).await; - // Give the endpoint driver a chance to transmit the rejection - // before this per-connection handler drops its stream. - tokio::time::sleep(Duration::from_millis(10)).await; + reject_authentication(&mut send, self.handshake_timeout).await; return Err(TransportError::AuthenticationRejected( RejectReason::AuthenticationFailed, )); @@ -554,8 +657,7 @@ impl QuicServer { input.extend_from_slice(id_bytes); let expected = auth::hmac_sha256(secret, &input); if !bool::from(expected.as_slice().ct_eq(&response[nonce_at + 16..])) { - let _ = send_result(&mut send, 1, 0, self.handshake_timeout).await; - tokio::time::sleep(Duration::from_millis(10)).await; + reject_authentication(&mut send, self.handshake_timeout).await; return Err(TransportError::AuthenticationRejected( RejectReason::AuthenticationFailed, )); @@ -564,12 +666,15 @@ impl QuicServer { .max_datagram_size() .ok_or_else(|| TransportError::Io(io_invalid("QUIC DATAGRAM is not negotiated")))?; let max_datagram = self.max_datagram.min(quic_max).min(u16::MAX as usize); + let admission = pending.promote(); send_result(&mut send, 0, max_datagram as u16, self.handshake_timeout).await?; debug!(client_id=%id, remote_addr=%connection.remote_address(), max_datagram, "QUIC client authenticated"); Ok(QuicDatagramSession::new( connection, self.endpoint.clone(), max_datagram, + Some(id.to_owned()), + Some(admission), )) } @@ -598,3 +703,12 @@ async fn send_result( .map_err(quinn_io)?; send.finish().map_err(quinn_io) } + +async fn reject_authentication(send: &mut quinn::SendStream, duration: Duration) { + // Wait for the client to consume the rejection (or stop reading) so the + // connection is not dropped before the reliable control result is sent. + // The wait is bounded for peers that abandon the handshake. + if send_result(send, 1, 0, duration).await.is_ok() { + let _ = timeout(duration, send.stopped()).await; + } +} diff --git a/tests/control.rs b/tests/control.rs index c498d68..9234388 100644 --- a/tests/control.rs +++ b/tests/control.rs @@ -1,26 +1,38 @@ -#[path = "../src/control.rs"] -mod control; +use wgcloak::control; use std::os::unix::fs::PermissionsExt; use tempfile::tempdir; -use tokio::time::{sleep, Duration}; #[tokio::test] async fn control_commands_and_socket_permissions() { let dir = tempdir().unwrap(); - let path = dir.path().join("control.sock"); + let runtime_dir = dir.path().join("runtime"); + let path = runtime_dir.join("control.sock"); let state = control::ControlState::default(); + state.enable_reconnect(); state.set_status("ready").await; state.set_stats_json("{\"packets\":3}").await; let server = control::ControlServer::bind(&path, state.clone()) .await .unwrap(); + assert_eq!( + std::fs::metadata(&runtime_dir) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700 + ); assert_eq!( std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, 0o600 ); let task = tokio::spawn(server.run()); - sleep(Duration::from_millis(10)).await; + let second_bind = control::ControlServer::bind(&path, control::ControlState::default()).await; + assert!(matches!( + second_bind, + Err(error) if error.kind() == std::io::ErrorKind::AddrInUse + )); assert!(control::send_command(&path, "status") .await .unwrap() @@ -33,13 +45,56 @@ async fn control_commands_and_socket_permissions() { .await .unwrap() .contains("true")); - assert!(control::send_command(&path, "reload") - .await - .unwrap() - .contains("true")); + let reload = control::send_command(&path, "reload").await.unwrap(); + assert!(reload.contains("\"ok\":false")); + assert!(reload.contains("not supported")); assert!(control::send_command(&path, "shutdown") .await .unwrap() .contains("true")); task.await.unwrap().unwrap(); } + +#[tokio::test] +async fn replaces_only_a_verified_stale_socket() { + let dir = tempdir().unwrap(); + let private_dir = dir.path().join("runtime"); + std::fs::create_dir(&private_dir).unwrap(); + std::fs::set_permissions(&private_dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + let path = private_dir.join("stale.sock"); + let stale = std::os::unix::net::UnixListener::bind(&path).unwrap(); + drop(stale); + + let server = control::ControlServer::bind(&path, control::ControlState::default()) + .await + .expect("verified stale socket should be replaced"); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + drop(server); + assert!(!path.exists()); +} + +#[tokio::test] +async fn refuses_to_replace_a_non_socket_path() { + let dir = tempdir().unwrap(); + let private_dir = dir.path().join("runtime"); + std::fs::create_dir(&private_dir).unwrap(); + std::fs::set_permissions(&private_dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + let path = private_dir.join("control.sock"); + std::fs::write(&path, b"keep this file").unwrap(); + let result = control::ControlServer::bind(&path, control::ControlState::default()).await; + assert!(matches!( + result, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists + )); + assert_eq!(std::fs::read(&path).unwrap(), b"keep this file"); +} + +#[test] +fn default_socket_path_never_targets_the_shared_tmp_socket() { + let path = control::default_socket_path(); + assert_eq!(path.file_name().unwrap(), "wgcloak.sock"); + assert_ne!(path.parent().unwrap(), std::path::Path::new("/tmp")); +} diff --git a/tests/direct_udp.rs b/tests/direct_udp.rs index 1593e1f..d673d8b 100644 --- a/tests/direct_udp.rs +++ b/tests/direct_udp.rs @@ -1,13 +1,4 @@ -#[path = "../src/config.rs"] -mod config; -#[path = "../src/datagram.rs"] -mod datagram; -#[path = "../src/protocol/mod.rs"] -mod protocol; -#[path = "../src/stats.rs"] -mod stats; -#[path = "../src/transport/mod.rs"] -mod transport; +use wgcloak::{config, protocol, transport}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -16,7 +7,7 @@ use bytes::Bytes; use config::Config; use tokio::{net::UdpSocket, time::timeout}; use transport::{ - direct_udp::{DirectUdpServer, DirectUdpSession}, + direct_udp::{DirectUdpServer, DirectUdpServerHandle, DirectUdpSession}, TransportError, TransportSession, }; @@ -51,6 +42,19 @@ fn multi_configs( (server, client("one"), client("two")) } +async fn wait_for_active_sessions(server: &DirectUdpServerHandle, expected: usize) { + timeout(Duration::from_secs(2), async { + loop { + if server.active_sessions().await == expected { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("active session count did not become {expected}")); +} + #[tokio::test] async fn direct_udp_round_trip_and_auth_rejection() { let backend = UdpSocket::bind("127.0.0.1:0").await.unwrap(); @@ -66,6 +70,11 @@ async fn direct_udp_round_trip_and_auth_rejection() { backend.send_to(&buf[..len], peer).await.unwrap(); }); let session = DirectUdpSession::connect(&client_config).await.unwrap(); + assert_eq!( + session.authenticated_keepalive_interval(), + Some(Duration::from_secs(15)) + ); + session.send_keepalive().await.unwrap(); session .send_datagram(Bytes::from_static(b"opaque")) .await @@ -99,19 +108,42 @@ async fn direct_udp_enforces_payload_limit_and_expires_idle_session() { let (mut server_config, client_config) = configs(45782, backend_addr.port()); server_config.session.idle_timeout_secs = 1; let server = DirectUdpServer::bind(server_config).await.unwrap(); - let stats = server.stats(); + let observer = server.handle(); + let stats = observer.stats(); let server_task = tokio::spawn(server.run()); let session = DirectUdpSession::connect(&client_config).await.unwrap(); + wait_for_active_sessions(&observer, 1).await; assert!(matches!( session.send_datagram(Bytes::from(vec![0u8; 1201])).await, Err(TransportError::Oversized { maximum: 1200 }) )); tokio::time::sleep(Duration::from_secs(3)).await; assert_eq!(stats.snapshot().expired_sessions, 1); + wait_for_active_sessions(&observer, 0).await; session.close().await.unwrap(); server_task.abort(); } +#[tokio::test] +async fn direct_udp_observer_tracks_authenticated_session_close() { + let backend = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let backend_addr = backend.local_addr().unwrap(); + let (server_config, client_config) = configs(45787, backend_addr.port()); + let server = DirectUdpServer::bind(server_config).await.unwrap(); + let observer = server.handle(); + let stats = observer.stats(); + let server_task = tokio::spawn(server.run()); + + let session = DirectUdpSession::connect(&client_config).await.unwrap(); + wait_for_active_sessions(&observer, 1).await; + assert_eq!(stats.snapshot().expired_sessions, 0); + + session.close().await.unwrap(); + wait_for_active_sessions(&observer, 0).await; + assert_eq!(stats.snapshot().expired_sessions, 0); + server_task.abort(); +} + #[tokio::test] async fn concurrent_clients_use_independent_backend_sockets() { let backend = UdpSocket::bind("127.0.0.1:0").await.unwrap(); @@ -178,6 +210,14 @@ async fn malformed_and_unauthenticated_packets_do_not_kill_server() { let raw = UdpSocket::bind("127.0.0.1:0").await.unwrap(); raw.send_to(&[1, 2, 3], "127.0.0.1:45784").await.unwrap(); raw.send_to(&[0u8; 64], "127.0.0.1:45784").await.unwrap(); + let oversized = protocol::DataPacket::new(1, 1, vec![0u8; 1201], [0; 16]) + .encode() + .unwrap(); + raw.send_to(&oversized, "127.0.0.1:45784").await.unwrap(); + let unexpected = protocol::RejectPacket::new(protocol::RejectReason::AuthenticationFailed) + .encode() + .unwrap(); + raw.send_to(&unexpected, "127.0.0.1:45784").await.unwrap(); let mut wrong = client_config.clone(); wrong.client.secret.secret = Some("B".repeat(64)); assert!(matches!( @@ -201,7 +241,8 @@ async fn malformed_and_unauthenticated_packets_do_not_kill_server() { .unwrap(), Bytes::from_static(b"still-alive") ); - assert!(stats.snapshot().datagrams_dropped >= 3); + assert!(stats.snapshot().datagrams_dropped >= 5); + assert!(stats.snapshot().oversized_datagrams >= 1); session.close().await.unwrap(); backend_task.await.unwrap(); server_task.abort(); @@ -275,6 +316,274 @@ async fn authenticated_source_port_change_updates_return_peer() { server_task.abort(); } +#[tokio::test] +async fn identical_hello_retransmission_recovers_original_accept_and_session() { + let backend = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let backend_addr = backend.local_addr().unwrap(); + let (server_config, _) = configs(45788, backend_addr.port()); + let server = DirectUdpServer::bind(server_config).await.unwrap(); + let observer = server.handle(); + let server_task = tokio::spawn(server.run()); + let client = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let other_source = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let secret = base64::engine::general_purpose::STANDARD + .decode(SECRET) + .unwrap(); + let mut nonce = [0u8; 16]; + nonce[0] = 44; + let hello = protocol::HelloPacket::authenticated( + &secret, + Bytes::from_static(b"test"), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(), + nonce, + ) + .unwrap(); + let hello_bytes = hello.encode().unwrap(); + client + .send_to(&hello_bytes, "127.0.0.1:45788") + .await + .unwrap(); + let mut response = [0u8; 2048]; + let (len, _) = timeout(Duration::from_secs(2), client.recv_from(&mut response)) + .await + .unwrap() + .unwrap(); + let first_accept = match protocol::parse_packet(&response[..len]).unwrap() { + protocol::Packet::Accept(accept) => accept, + other => panic!("unexpected handshake response: {other:?}"), + }; + + client + .send_to(&hello_bytes, "127.0.0.1:45788") + .await + .unwrap(); + let (len, _) = timeout(Duration::from_secs(2), client.recv_from(&mut response)) + .await + .unwrap() + .unwrap(); + let retry_accept = match protocol::parse_packet(&response[..len]).unwrap() { + protocol::Packet::Accept(accept) => accept, + other => panic!("identical HELLO should recover ACCEPT, got {other:?}"), + }; + assert_eq!(retry_accept, first_accept); + + wait_for_active_sessions(&observer, 1).await; + other_source + .send_to(&hello_bytes, "127.0.0.1:45788") + .await + .unwrap(); + let (len, _) = timeout( + Duration::from_secs(2), + other_source.recv_from(&mut response), + ) + .await + .unwrap() + .unwrap(); + assert!(matches!( + protocol::parse_packet(&response[..len]), + Ok(protocol::Packet::Reject(protocol::RejectPacket { + reason: protocol::RejectReason::ReplayDetected, + .. + })) + )); + + let incompatible = protocol::HelloPacket::authenticated( + &secret, + Bytes::from_static(b"test"), + hello.timestamp + 1, + nonce, + ) + .unwrap(); + client + .send_to(&incompatible.encode().unwrap(), "127.0.0.1:45788") + .await + .unwrap(); + let (len, _) = timeout(Duration::from_secs(2), client.recv_from(&mut response)) + .await + .unwrap() + .unwrap(); + assert!(matches!( + protocol::parse_packet(&response[..len]), + Ok(protocol::Packet::Reject(protocol::RejectPacket { + reason: protocol::RejectReason::ReplayDetected, + .. + })) + )); + + let session_key = protocol::derive_session_key( + &secret, + &nonce, + &first_accept.server_nonce, + first_accept.session_id, + ); + let (peer_tx, mut peer_rx) = tokio::sync::mpsc::unbounded_channel(); + let backend_task = tokio::spawn(async move { + let mut buffer = [0u8; 2048]; + for _ in 0..2 { + let (len, peer) = backend.recv_from(&mut buffer).await.unwrap(); + peer_tx.send(peer).unwrap(); + backend.send_to(&buffer[..len], peer).await.unwrap(); + } + }); + for (sequence, payload) in [(1, b"one".as_slice()), (2, b"two".as_slice())] { + let packet = protocol::DataPacket::authenticated( + &session_key, + first_accept.session_id, + sequence, + Bytes::copy_from_slice(payload), + ); + client + .send_to(&packet.encode().unwrap(), "127.0.0.1:45788") + .await + .unwrap(); + let (len, _) = timeout(Duration::from_secs(2), client.recv_from(&mut response)) + .await + .unwrap() + .unwrap(); + assert!(matches!( + protocol::parse_packet(&response[..len]), + Ok(protocol::Packet::Data(_)) + )); + } + backend_task.await.unwrap(); + assert_eq!(peer_rx.recv().await, peer_rx.recv().await); + assert_eq!(observer.active_sessions().await, 1); + + let close = protocol::SessionPacket::authenticated( + protocol::PacketType::Close, + &session_key, + first_accept.session_id, + 3, + ); + client + .send_to(&close.encode().unwrap(), "127.0.0.1:45788") + .await + .unwrap(); + wait_for_active_sessions(&observer, 0).await; + client + .send_to(&hello_bytes, "127.0.0.1:45788") + .await + .unwrap(); + let (len, _) = timeout(Duration::from_secs(2), client.recv_from(&mut response)) + .await + .unwrap() + .unwrap(); + assert!(matches!( + protocol::parse_packet(&response[..len]), + Ok(protocol::Packet::Reject(protocol::RejectPacket { + reason: protocol::RejectReason::ReplayDetected, + .. + })) + )); + server_task.abort(); +} + +#[tokio::test] +async fn flags_are_authenticated_and_invalid_close_cannot_remove_session() { + let backend = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let backend_addr = backend.local_addr().unwrap(); + let (server_config, _) = configs(45789, backend_addr.port()); + let server = DirectUdpServer::bind(server_config).await.unwrap(); + let observer = server.handle(); + let server_task = tokio::spawn(server.run()); + let client = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let secret = base64::engine::general_purpose::STANDARD + .decode(SECRET) + .unwrap(); + let mut nonce = [0u8; 16]; + nonce[0] = 45; + let hello = protocol::HelloPacket::authenticated( + &secret, + Bytes::from_static(b"test"), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(), + nonce, + ) + .unwrap(); + client + .send_to(&hello.encode().unwrap(), "127.0.0.1:45789") + .await + .unwrap(); + let mut response = [0u8; 2048]; + let (len, _) = timeout(Duration::from_secs(2), client.recv_from(&mut response)) + .await + .unwrap() + .unwrap(); + let accept = match protocol::parse_packet(&response[..len]).unwrap() { + protocol::Packet::Accept(accept) => accept, + other => panic!("unexpected handshake response: {other:?}"), + }; + let key = + protocol::derive_session_key(&secret, &nonce, &accept.server_nonce, accept.session_id); + + let data = protocol::DataPacket::authenticated(&key, accept.session_id, 1, b"valid".as_slice()); + let mut mutated_flags = data.encode().unwrap(); + mutated_flags[2] = 1; + client + .send_to(&mutated_flags, "127.0.0.1:45789") + .await + .unwrap(); + assert!(timeout( + Duration::from_millis(100), + backend.recv_from(&mut [0u8; 32]) + ) + .await + .is_err()); + + let invalid_close = protocol::SessionPacket::authenticated( + protocol::PacketType::Close, + &key, + accept.session_id, + 1, + ); + let mut mutated_close_flags = invalid_close.encode().unwrap(); + mutated_close_flags[2] = 1; + client + .send_to(&mutated_close_flags, "127.0.0.1:45789") + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(observer.active_sessions().await, 1); + + let valid_data = protocol::DataPacket::authenticated( + &key, + accept.session_id, + 1, + Bytes::from_static(b"still-open"), + ); + client + .send_to(&valid_data.encode().unwrap(), "127.0.0.1:45789") + .await + .unwrap(); + let mut backend_buffer = [0u8; 64]; + let (len, _) = timeout( + Duration::from_secs(2), + backend.recv_from(&mut backend_buffer), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(&backend_buffer[..len], b"still-open"); + + let valid_close = protocol::SessionPacket::authenticated( + protocol::PacketType::Close, + &key, + accept.session_id, + 2, + ); + client + .send_to(&valid_close.encode().unwrap(), "127.0.0.1:45789") + .await + .unwrap(); + wait_for_active_sessions(&observer, 0).await; + server_task.abort(); +} + #[tokio::test] async fn unavailable_backend_does_not_terminate_listener() { let (server_config, client_config) = configs(45786, 9); diff --git a/tests/http3.rs b/tests/http3.rs index 87fef54..d20422d 100644 --- a/tests/http3.rs +++ b/tests/http3.rs @@ -1,12 +1,10 @@ -#[path = "../src/transport/http3.rs"] -mod http3; - use std::collections::HashMap; -use http3::{ - decode_capsule, decode_http_datagram, encode_auth_challenge, encode_auth_response, - encode_capsule, encode_http_datagram, validate_connect_udp_request, AuthChallenge, - ConnectUdpRequest, Http3Error, CONNECT_UDP_CONTEXT_ID, +use wgcloak::transport::http3; +use wgcloak::transport::http3::{ + decode_auth_challenge, decode_capsule, decode_http_datagram, encode_auth_challenge, + encode_auth_response, encode_capsule, encode_http_datagram, validate_connect_udp_request, + AuthChallenge, ConnectUdpRequest, Http3Error, CONNECT_UDP_CONTEXT_ID, }; #[test] @@ -78,6 +76,10 @@ fn authentication_is_completed_before_session_activation() { let nonce = [5u8; 16]; let secret = [7u8; 32]; let challenge_capsule = encode_auth_challenge(&challenge).unwrap(); + assert_eq!( + decode_auth_challenge(&challenge_capsule).unwrap(), + challenge + ); assert_eq!( decode_capsule(&challenge_capsule).unwrap().0, http3::AUTH_CHALLENGE_CAPSULE @@ -94,3 +96,26 @@ fn authentication_is_completed_before_session_activation() { assert!(http3::is_auth_accept(&http3::encode_auth_accept().unwrap()).unwrap()); assert!(!http3::is_auth_accept(&http3::encode_auth_reject().unwrap()).unwrap()); } + +#[test] +fn datagram_limit_includes_http_context_and_request_stream_overhead() { + use h3::quic::StreamId; + + let limit = http3::maximum_application_datagram( + 1200, + StreamId::try_from(0).unwrap(), + CONNECT_UDP_CONTEXT_ID, + 1200, + ) + .unwrap(); + assert_eq!(limit, 1198); + assert!(matches!( + http3::maximum_application_datagram( + 2, + StreamId::try_from(0).unwrap(), + CONNECT_UDP_CONTEXT_ID, + 1200 + ), + Ok(0) + )); +} diff --git a/tests/http3_integration.rs b/tests/http3_integration.rs new file mode 100644 index 0000000..c7beca1 --- /dev/null +++ b/tests/http3_integration.rs @@ -0,0 +1,272 @@ +use std::{collections::HashMap, fs, process::Command, sync::Arc, time::Duration}; + +use base64::Engine; +use bytes::Bytes; +use quinn::{ClientConfig, Endpoint, ServerConfig}; +use rustls::pki_types::{pem::PemObject, CertificateDer, PrivateKeyDer}; +use tokio::time::timeout; +use wgcloak::config; +use wgcloak::transport::{ + http3::{Http3ClientConnection, Http3Error, Http3Server, Http3ServerConnection}, + AdmissionController, TransportSession, +}; + +const SECRET: &str = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; +const WRONG_SECRET: &str = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"; + +fn test_endpoints() -> ( + tempfile::TempDir, + Endpoint, + ClientConfig, + std::net::SocketAddr, +) { + let directory = tempfile::tempdir().unwrap(); + let certificate = directory.path().join("localhost-cert.pem"); + let private_key = directory.path().join("localhost-key.pem"); + let result = Command::new("openssl") + .args([ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + private_key.to_str().unwrap(), + "-out", + certificate.to_str().unwrap(), + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + "-addext", + "basicConstraints=critical,CA:FALSE", + "-addext", + "keyUsage=critical,digitalSignature,keyEncipherment", + "-addext", + "extendedKeyUsage=serverAuth", + "-days", + "1", + ]) + .output() + .unwrap(); + assert!( + result.status.success(), + "openssl failed: {}", + String::from_utf8_lossy(&result.stderr) + ); + + let cert_pem = fs::read(&certificate).unwrap(); + let certs: Vec> = CertificateDer::pem_slice_iter(&cert_pem) + .collect::>() + .unwrap(); + let key_pem = fs::read(&private_key).unwrap(); + let key = PrivateKeyDer::from_pem_slice(&key_pem).unwrap(); + let mut server_config = ServerConfig::with_single_cert(certs.clone(), key).unwrap(); + let mut server_transport = quinn::TransportConfig::default(); + server_transport.datagram_send_buffer_size(1024 * 1024); + server_config.transport_config(Arc::new(server_transport)); + let server = Endpoint::server(server_config, "127.0.0.1:0".parse().unwrap()).unwrap(); + let server_address = server.local_addr().unwrap(); + + let mut roots = rustls::RootCertStore::empty(); + roots.add(certs[0].clone()).unwrap(); + let mut client_config = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + let mut client_transport = quinn::TransportConfig::default(); + client_transport.datagram_send_buffer_size(1024 * 1024); + client_config.transport_config(Arc::new(client_transport)); + (directory, server, client_config, server_address) +} + +async fn connect_client(address: std::net::SocketAddr, config: ClientConfig) -> quinn::Connection { + let mut endpoint = Endpoint::client("[::]:0".parse().unwrap()).unwrap(); + endpoint.set_default_client_config(config); + endpoint + .connect(address, "localhost") + .unwrap() + .await + .unwrap() +} + +#[tokio::test] +async fn connect_udp_authenticates_then_exchanges_contextual_http_datagrams() { + let (_certificate_dir, server_endpoint, client_tls, address) = test_endpoints(); + let mut clients = HashMap::new(); + clients.insert( + "laptop".to_owned(), + base64::engine::general_purpose::STANDARD + .decode(SECRET) + .unwrap(), + ); + let server_task = tokio::spawn(async move { + let incoming = server_endpoint.accept().await.unwrap(); + let quic_connection = incoming.await.unwrap(); + Http3ServerConnection::new(quic_connection) + .await + .unwrap() + .accept_connect_udp("/tunnel", &clients, 1200, Duration::from_secs(3)) + .await + .unwrap() + }); + + let quic_connection = connect_client(address, client_tls).await; + let client = Http3ClientConnection::new(quic_connection).await.unwrap(); + let client_session = client + .connect_udp( + &format!("localhost:{}", address.port()), + "/tunnel", + "laptop", + &base64::engine::general_purpose::STANDARD + .decode(SECRET) + .unwrap(), + 1200, + Duration::from_secs(3), + ) + .await + .unwrap(); + let server_session = timeout(Duration::from_secs(3), server_task) + .await + .unwrap() + .unwrap(); + + assert_eq!( + client_session.max_datagram_size(), + server_session.max_datagram_size() + ); + assert!(client_session.max_datagram_size() < 1200); + client_session + .send_datagram(b"opaque-upstream") + .await + .unwrap(); + assert_eq!( + timeout(Duration::from_secs(2), server_session.recv_datagram()) + .await + .unwrap() + .unwrap(), + Bytes::from_static(b"opaque-upstream") + ); + server_session + .send_datagram(b"opaque-downstream") + .await + .unwrap(); + assert_eq!( + timeout(Duration::from_secs(2), client_session.recv_datagram()) + .await + .unwrap() + .unwrap(), + Bytes::from_static(b"opaque-downstream") + ); + let too_large = vec![0; client_session.max_datagram_size() + 1]; + assert!(matches!( + client_session.send_datagram(&too_large).await, + Err(Http3Error::Oversized { .. }) + )); + client_session.close(Duration::from_secs(2)).await.unwrap(); + assert!(matches!( + client_session.send_datagram(b"after-close").await, + Err(Http3Error::Closed) + )); +} + +#[tokio::test] +async fn connect_udp_rejects_bad_secret_before_returning_session() { + let (_certificate_dir, server_endpoint, client_tls, address) = test_endpoints(); + let mut clients = HashMap::new(); + clients.insert( + "laptop".to_owned(), + base64::engine::general_purpose::STANDARD + .decode(SECRET) + .unwrap(), + ); + let server_task = tokio::spawn(async move { + let incoming = server_endpoint.accept().await.unwrap(); + let quic_connection = incoming.await.unwrap(); + Http3ServerConnection::new(quic_connection) + .await + .unwrap() + .accept_connect_udp("/tunnel", &clients, 1200, Duration::from_secs(3)) + .await + }); + + let quic_connection = connect_client(address, client_tls).await; + let client = Http3ClientConnection::new(quic_connection).await.unwrap(); + let client_result = client + .connect_udp( + &format!("localhost:{}", address.port()), + "/tunnel", + "laptop", + &base64::engine::general_purpose::STANDARD + .decode(WRONG_SECRET) + .unwrap(), + 1200, + Duration::from_secs(3), + ) + .await; + assert!( + matches!(client_result, Err(Http3Error::AuthenticationRejected)), + "unexpected client error: {:?}", + client_result.as_ref().err() + ); + let server_result = timeout(Duration::from_secs(3), server_task) + .await + .unwrap() + .unwrap(); + assert!( + matches!(server_result, Err(Http3Error::AuthenticationRejected)), + "unexpected server error: {:?}", + server_result.as_ref().err() + ); +} + +#[tokio::test] +async fn http3_server_admission_permit_tracks_authenticated_session_lifetime() { + let (_certificate_dir, endpoint, client_tls, address) = test_endpoints(); + let mut clients = HashMap::new(); + clients.insert( + "laptop".to_owned(), + base64::engine::general_purpose::STANDARD + .decode(SECRET) + .unwrap(), + ); + let limits = config::LimitsConfig { + max_sessions: 1, + max_sessions_per_ip: 1, + ..config::LimitsConfig::default() + }; + let admission = AdmissionController::new(&limits); + let server = Arc::new(Http3Server::from_endpoint_with_admission( + endpoint, + "/tunnel", + clients, + 1200, + Duration::from_secs(3), + admission.clone(), + )); + let accept_server = server.clone(); + let server_task = tokio::spawn(async move { accept_server.accept_session().await.unwrap() }); + + let quic_connection = connect_client(address, client_tls).await; + let client = Http3ClientConnection::new(quic_connection).await.unwrap(); + let client_session = client + .connect_udp( + &format!("localhost:{}", address.port()), + "/tunnel", + "laptop", + &base64::engine::general_purpose::STANDARD + .decode(SECRET) + .unwrap(), + 1200, + Duration::from_secs(3), + ) + .await + .unwrap(); + let server_session = timeout(Duration::from_secs(3), server_task) + .await + .unwrap() + .unwrap(); + + assert_eq!(admission.active_sessions(), 1); + assert_eq!(admission.pending_handshakes(), 0); + server_session.close().await.unwrap(); + assert_eq!(admission.active_sessions(), 0); + client_session.close(Duration::from_secs(2)).await.unwrap(); +} diff --git a/tests/http3_runtime.rs b/tests/http3_runtime.rs new file mode 100644 index 0000000..f6e7f4c --- /dev/null +++ b/tests/http3_runtime.rs @@ -0,0 +1,394 @@ +use std::{ + fs, net::SocketAddr, os::unix::fs::PermissionsExt, path::Path, process::Command, sync::Arc, + time::Duration, +}; + +use rustls::pki_types::pem::PemObject; +use tempfile::TempDir; +use tokio::{ + net::UdpSocket, + sync::mpsc, + time::{sleep, timeout}, +}; +use wgcloak::{ + client, + config::{self, Config}, + control, server, stats, transport, + transport::{Http3ClientConnector, TransportConnector, TransportSession}, +}; + +const SECRET_ONE: &str = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; +const SECRET_TWO: &str = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"; +const WRONG_SECRET: &str = "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC"; + +fn free_udp_port() -> u16 { + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + socket.local_addr().unwrap().port() +} + +fn private_runtime_dir(root: &Path) -> std::path::PathBuf { + let path = root.join("runtime"); + fs::create_dir(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); + path +} + +fn server_config( + directory: &TempDir, + backend: SocketAddr, + direct_port: u16, + http3_port: u16, +) -> config::ServerConfig { + let certificate = directory.path().join("server-cert.pem"); + let private_key = directory.path().join("server-key.pem"); + let generated = Command::new("openssl") + .args(["req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout"]) + .arg(&private_key) + .args(["-out"]) + .arg(&certificate) + .args([ + "-subj", + "/CN=127.0.0.1", + "-addext", + "subjectAltName=IP:127.0.0.1", + "-addext", + "basicConstraints=critical,CA:FALSE", + "-addext", + "keyUsage=critical,digitalSignature,keyEncipherment", + "-addext", + "extendedKeyUsage=serverAuth", + "-days", + "1", + ]) + .output() + .expect("openssl is required for the HTTP/3 runtime integration test"); + assert!( + generated.status.success(), + "openssl certificate generation failed: {}", + String::from_utf8_lossy(&generated.stderr) + ); + + Config::from_str(&format!( + "[server]\nlisten='127.0.0.1'\n[backend]\ntarget='{backend}'\n[transports.direct_udp]\nenabled=true\nport={direct_port}\n[transports.quic]\nenabled=false\nport={}\n[transports.http3]\nenabled=true\nport={http3_port}\npath='/tunnel'\n[transports.tls_tcp]\nenabled=true\nport={direct_port}\n[tls]\ncertificate='{}'\nprivate_key='{}'\n[[auth.clients]]\nid='one'\nsecret='{SECRET_ONE}'\n[[auth.clients]]\nid='two'\nsecret='{SECRET_TWO}'\n[limits]\nhandshake_timeout_ms=3000\n", + free_udp_port(), + certificate.display(), + private_key.display() + )) + .unwrap() + .into_server() + .unwrap() +} + +fn client_config( + id: &str, + secret: &str, + http3_port: u16, + direct_probe_port: u16, + local_port: u16, +) -> config::ClientConfig { + Config::from_str(&format!( + "[client]\nserver='127.0.0.1'\nid='{id}'\nsecret='{secret}'\n[local]\nlisten='127.0.0.1:{local_port}'\n[transport]\nmode='auto'\n[transport.auto]\norder=['direct_udp','http3']\nprobe_timeout_ms=700\nretries=1\n[transport.direct_udp]\nport={direct_probe_port}\n[transport.http3]\nport={http3_port}\npath='/tunnel'\n[network]\nmax_datagram_size=1200\n" + )) + .unwrap() + .into_client() + .unwrap() +} + +fn test_client_tls(certificate: &[u8]) -> quinn::ClientConfig { + let _ = rustls::crypto::ring::default_provider().install_default(); + let certs: Vec<_> = rustls::pki_types::CertificateDer::pem_slice_iter(certificate) + .collect::>() + .unwrap(); + let mut roots = rustls::RootCertStore::empty(); + roots.add(certs[0].clone()).unwrap(); + let mut tls = rustls::ClientConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + tls.alpn_protocols = vec![b"h3".to_vec()]; + let crypto = quinn::crypto::rustls::QuicClientConfig::try_from(tls).unwrap(); + quinn::ClientConfig::new(Arc::new(crypto)) +} + +async fn wait_control_contains(path: &std::path::Path, command: &str, needle: &str) { + let mut last_response = None; + let result = timeout(Duration::from_secs(6), async { + loop { + if let Ok(response) = control::send_command(path, command).await { + if response.contains(needle) { + return; + } + last_response = Some(response); + } + sleep(Duration::from_millis(40)).await; + } + }) + .await; + assert!( + result.is_ok(), + "control {command} did not contain {needle:?}; latest response: {last_response:?}" + ); +} + +#[tokio::test] +async fn auto_http3_runtime_relays_isolated_clients_and_rejects_bad_secret() { + if Command::new("openssl").arg("version").output().is_err() { + return; + } + let directory = tempfile::tempdir().unwrap(); + let runtime_dir = private_runtime_dir(directory.path()); + let backend = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let backend_addr = backend.local_addr().unwrap(); + let (peer_tx, mut peer_rx) = mpsc::unbounded_channel(); + let backend_task = tokio::spawn(async move { + let mut packet = vec![0u8; u16::MAX as usize]; + while let Ok((len, peer)) = backend.recv_from(&mut packet).await { + let _ = peer_tx.send(peer); + let _ = backend.send_to(&packet[..len], peer).await; + } + }); + + let certificate = directory.path().join("server-cert.pem"); + let direct_server_port = free_udp_port(); + let http3_port = free_udp_port(); + let server = server_config(&directory, backend_addr, direct_server_port, http3_port); + let server_control = runtime_dir.join("server.sock"); + let server_control_for_task = server_control.clone(); + let server_task = tokio::spawn(server::run_with_control_path( + server, + server_control_for_task, + )); + wait_control_contains( + &server_control, + "status", + "transports=direct_udp,http3,tls_tcp", + ) + .await; + + let certificate_bytes = fs::read(certificate).unwrap(); + let client_tls = test_client_tls(&certificate_bytes); + let direct_probe_port = free_udp_port(); + assert_ne!(direct_probe_port, direct_server_port); + + let local_one = free_udp_port(); + let local_two = free_udp_port(); + let app_one = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let app_two = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let client_one_config = + client_config("one", SECRET_ONE, http3_port, direct_probe_port, local_one); + let client_two_config = + client_config("two", SECRET_TWO, http3_port, direct_probe_port, local_two); + let client_one_control = runtime_dir.join("client-one.sock"); + let client_two_control = runtime_dir.join("client-two.sock"); + let client_one_task = tokio::spawn(client::run_with_http3_connector_and_control_path( + client_one_config.clone(), + client_one_control.clone(), + Some(Arc::new(Http3ClientConnector::with_client_config( + client_one_config, + client_tls.clone(), + )) as Arc), + )); + let client_two_task = tokio::spawn(client::run_with_http3_connector_and_control_path( + client_two_config.clone(), + client_two_control.clone(), + Some(Arc::new(Http3ClientConnector::with_client_config( + client_two_config, + client_tls.clone(), + )) as Arc), + )); + + wait_control_contains(&server_control, "status", "active_sessions=2").await; + app_one + .send_to(b"client-one", ("127.0.0.1", local_one)) + .await + .unwrap(); + app_two + .send_to(b"client-two", ("127.0.0.1", local_two)) + .await + .unwrap(); + let mut response = [0u8; 64]; + let (len_one, _) = timeout(Duration::from_secs(4), app_one.recv_from(&mut response)) + .await + .expect("first HTTP/3 echo timed out") + .unwrap(); + assert_eq!(&response[..len_one], b"client-one"); + let (len_two, _) = timeout(Duration::from_secs(4), app_two.recv_from(&mut response)) + .await + .expect("second HTTP/3 echo timed out") + .unwrap(); + assert_eq!(&response[..len_two], b"client-two"); + let peer_one = timeout(Duration::from_secs(2), peer_rx.recv()) + .await + .unwrap() + .unwrap(); + let peer_two = timeout(Duration::from_secs(2), peer_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_ne!( + peer_one, peer_two, + "each tunnel must own its backend UDP socket" + ); + + let bad_local = free_udp_port(); + let bad_client_config = client_config( + "one", + WRONG_SECRET, + http3_port, + direct_probe_port, + bad_local, + ); + let bad_control = runtime_dir.join("client-bad.sock"); + let bad_task = tokio::spawn(client::run_with_http3_connector_and_control_path( + bad_client_config.clone(), + bad_control.clone(), + Some(Arc::new(Http3ClientConnector::with_client_config( + bad_client_config, + client_tls, + )) as Arc), + )); + let bad_result = timeout(Duration::from_secs(6), bad_task) + .await + .expect("bad-secret client should finish its rejected startup") + .unwrap(); + assert!(bad_result.is_err(), "wrong project secret must be rejected"); + wait_control_contains(&server_control, "stats", "\"authentication_rejections\":1").await; + let aggregate = control::send_command(&server_control, "stats") + .await + .unwrap(); + assert!(aggregate.contains("\"transport\":\"multi\"")); + assert!(aggregate.contains("\"direct_udp\":{")); + assert!(aggregate.contains("\"http3\":{")); + assert!(aggregate.contains("\"tls_tcp\":{")); + + for path in [&client_one_control, &client_two_control] { + control::send_command(path, "shutdown").await.unwrap(); + } + timeout(Duration::from_secs(4), client_one_task) + .await + .unwrap() + .unwrap() + .unwrap(); + timeout(Duration::from_secs(4), client_two_task) + .await + .unwrap() + .unwrap() + .unwrap(); + control::send_command(&server_control, "shutdown") + .await + .unwrap(); + timeout(Duration::from_secs(5), server_task) + .await + .unwrap() + .unwrap() + .unwrap(); + backend_task.abort(); +} + +#[tokio::test] +async fn conflicting_udp_listeners_are_rejected() { + let parsed = Config::from_str(&format!( + "[server]\nlisten='127.0.0.1'\n[backend]\ntarget='127.0.0.1:9'\n[transports.direct_udp]\nenabled=true\nport=54001\n[transports.quic]\nenabled=true\nport=54001\n[transports.http3]\nenabled=false\n[transports.tls_tcp]\nenabled=true\nport=54001\n[tls]\ncertificate='placeholder-cert'\nprivate_key='placeholder-key'\n[[auth.clients]]\nid='one'\nsecret='{SECRET_ONE}'\n" + )) + .unwrap() + .into_server() + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let runtime_dir = private_runtime_dir(directory.path()); + // Validation happens before binding any listener and reports only the + // incompatible same-protocol UDP bind; TCP is allowed to reuse the number. + let result = server::run_with_control_path(parsed, runtime_dir.join("conflict.sock")).await; + assert!(result.unwrap_err().to_string().contains("UDP listeners")); +} + +#[tokio::test] +async fn control_stats_and_status_merge_multiple_transport_snapshots() { + let directory = tempfile::tempdir().unwrap(); + let path = private_runtime_dir(directory.path()).join("aggregate.sock"); + let state = control::ControlState::default(); + state + .set_status("listening; transports=direct_udp,http3") + .await; + let direct = stats::StatsSnapshot { + datagrams_sent: 2, + bytes_sent: 40, + ..stats::StatsSnapshot::default() + }; + let http3 = stats::StatsSnapshot { + datagrams_received: 3, + authentication_rejections: 1, + ..stats::StatsSnapshot::default() + }; + state.set_transport_stats("direct_udp", 1, direct).await; + state.set_transport_stats("http3", 2, http3).await; + let server = control::ControlServer::bind(&path, state.clone()) + .await + .unwrap(); + let task = tokio::spawn(server.run()); + + let status = control::send_command(&path, "status").await.unwrap(); + assert!(status.contains("active_sessions=3")); + assert!(status.contains("active_sessions_by_transport=direct_udp:1,http3:2")); + let response = control::send_command(&path, "stats").await.unwrap(); + assert!(response.contains("\"datagrams_sent\":2")); + assert!(response.contains("\"bytes_sent\":40")); + assert!(response.contains("\"datagrams_received\":3")); + assert!(response.contains("\"authentication_rejections\":1")); + assert!(response.contains("\"direct_udp\":{")); + assert!(response.contains("\"http3\":{")); + + state.shutdown.cancel(); + timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap() + .unwrap(); +} + +#[tokio::test] +async fn direct_udp_runtime_cancellation_runs_session_cleanup() { + let port = free_udp_port(); + let server = Config::from_str(&format!( + "[server]\nlisten='127.0.0.1'\n[backend]\ntarget='127.0.0.1:9'\n[transports.direct_udp]\nenabled=true\nport={port}\n[transports.quic]\nenabled=false\n[transports.http3]\nenabled=false\n[transports.tls_tcp]\nenabled=false\n[[auth.clients]]\nid='lifecycle'\nsecret='{SECRET_ONE}'\n" + )) + .unwrap() + .into_server() + .unwrap(); + let client = Config::from_str(&format!( + "[client]\nserver='127.0.0.1'\nid='lifecycle'\nsecret='{SECRET_ONE}'\n[transport]\nmode='direct_udp'\n[transport.direct_udp]\nport={port}\n" + )) + .unwrap() + .into_client() + .unwrap(); + + let listener = transport::DirectUdpServer::bind(server).await.unwrap(); + let observer = listener.handle(); + let shutdown = tokio_util::sync::CancellationToken::new(); + let run = tokio::spawn(listener.run_with_shutdown(shutdown.clone())); + let session = timeout( + Duration::from_secs(3), + transport::direct_udp::DirectUdpSession::connect(&client), + ) + .await + .expect("direct UDP client authentication timed out") + .unwrap(); + timeout(Duration::from_secs(2), async { + while observer.active_sessions().await != 1 { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("server did not register the authenticated direct session"); + + shutdown.cancel(); + timeout(Duration::from_secs(2), run) + .await + .expect("direct listener did not complete graceful cleanup") + .unwrap() + .unwrap(); + assert_eq!(observer.active_sessions().await, 0); + session.close().await.unwrap(); +} diff --git a/tests/integration_transports.rs b/tests/integration_transports.rs index ef86446..9e3a256 100644 --- a/tests/integration_transports.rs +++ b/tests/integration_transports.rs @@ -1,22 +1,16 @@ -#[path = "../src/config.rs"] -mod config; -#[path = "../src/datagram.rs"] -mod datagram; -#[path = "../src/protocol/mod.rs"] -mod protocol; -#[path = "../src/stats.rs"] -mod stats; -#[path = "../src/transport/mod.rs"] -mod transport; - use std::{process::Command, sync::Arc, time::Duration}; use bytes::Bytes; -use config::Config; use rustls::RootCertStore; use tempfile::tempdir; use tokio::time::timeout; -use transport::{TlsTcpConnector, TlsTcpServer, TransportConnector, TransportSession}; +use wgcloak::{ + config::Config, + protocol, transport, + transport::{ + AdmissionController, TlsTcpConnector, TlsTcpServer, TransportConnector, TransportError, + }, +}; const SECRET: &str = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; @@ -29,7 +23,7 @@ fn free_port() -> u16 { } #[tokio::test] -async fn tls_tcp_authenticates_and_round_trips_opaque_datagrams() { +async fn tls_tcp_authenticates_round_trips_and_classifies_failures() { if Command::new("openssl").arg("version").output().is_err() { return; } @@ -45,6 +39,10 @@ async fn tls_tcp_authenticates_and_round_trips_opaque_datagrams() { "-nodes", "-subj", "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + "-addext", + "basicConstraints=critical,CA:FALSE", "-days", "1", "-keyout", @@ -61,7 +59,7 @@ async fn tls_tcp_authenticates_and_round_trips_opaque_datagrams() { ); let port = free_port(); let server_config = Config::from_str(&format!( - "[server]\nlisten = \"127.0.0.1\"\n[backend]\ntarget = \"127.0.0.1:51820\"\n[transports.direct_udp]\nenabled = false\n[transports.quic]\nenabled = false\n[transports.http3]\nenabled = false\n[transports.tls_tcp]\nenabled = true\nport = {port}\n[tls]\ncertificate = \"{}\"\nprivate_key = \"{}\"\n[[auth.clients]]\nid = \"test\"\nsecret = \"{SECRET}\"\n", + "[server]\nlisten = \"127.0.0.1\"\n[backend]\ntarget = \"127.0.0.1:51820\"\n[transports.direct_udp]\nenabled = false\n[transports.quic]\nenabled = false\n[transports.http3]\nenabled = false\n[transports.tls_tcp]\nenabled = true\nport = {port}\n[limits]\nhandshake_timeout_ms = 500\n[tls]\ncertificate = \"{}\"\nprivate_key = \"{}\"\n[[auth.clients]]\nid = \"test\"\nsecret = \"{SECRET}\"\n", cert.display(), key.display() )) @@ -69,23 +67,47 @@ async fn tls_tcp_authenticates_and_round_trips_opaque_datagrams() { .into_server() .unwrap(); let client_config = Config::from_str(&format!( - "[client]\nserver = \"localhost\"\nid = \"test\"\nsecret = \"{SECRET}\"\n[transport]\nmode = \"tls_tcp\"\n[transport.tls_tcp]\nport = {port}\n[network]\nmax_datagram_size = 1200\n" + "[client]\nserver = \"localhost\"\nid = \"test\"\nsecret = \"{SECRET}\"\n[transport]\nmode = \"tls_tcp\"\n[transport.tls_tcp]\nport = {port}\nkeepalive_secs = 1\n[network]\nmax_datagram_size = 1200\n" )) .unwrap() .into_client() .unwrap(); + let mismatch_config = client_config.clone(); let certs = transport::tls_tcp::load_certificates(&cert).unwrap(); let mut roots = RootCertStore::empty(); roots.add(certs[0].clone()).unwrap(); let client_tls = (*transport::tls_tcp::client_tls_config(roots)).clone(); - let server = Arc::new(TlsTcpServer::bind(&server_config).await.unwrap()); + let admission = AdmissionController::new(&server_config.limits); + let server = Arc::new( + TlsTcpServer::bind_with_admission(&server_config, admission.clone()) + .await + .unwrap(), + ); + let slow_server = server.clone(); + let slow_handshake = tokio::spawn(async move { slow_server.accept_session().await }); + let slow_peer = tokio::net::TcpStream::connect(server.local_addr().unwrap()) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; let accept = { let server = server.clone(); tokio::spawn(async move { server.accept_session().await }) }; - let client = TlsTcpConnector::with_client_config(client_config, client_tls); + let client = TlsTcpConnector::with_client_config(client_config, client_tls.clone()); let client_session = client.connect().await.unwrap(); let server_session = accept.await.unwrap().unwrap(); + assert_eq!( + client_session.authenticated_keepalive_interval(), + Some(Duration::from_secs(1)) + ); + assert_eq!(client_session.rtt(), None); + assert_eq!( + client_session.remote_addr().unwrap().ip(), + "127.0.0.1".parse::().unwrap() + ); + assert_eq!(admission.active_sessions(), 1); + assert_eq!(admission.pending_handshakes(), 1); + client_session.send_keepalive().await.unwrap(); client_session .send_datagram(Bytes::from_static(b"tls-opaque")) .await @@ -99,4 +121,59 @@ async fn tls_tcp_authenticates_and_round_trips_opaque_datagrams() { ); client_session.close().await.unwrap(); server_session.close().await.unwrap(); + assert!(matches!( + timeout(Duration::from_secs(1), slow_handshake) + .await + .unwrap() + .unwrap(), + Err(TransportError::HandshakeTimeout) + )); + drop(slow_peer); + assert_eq!(admission.active_sessions(), 0); + assert_eq!(admission.pending_handshakes(), 0); + + let mut wrong_credentials = mismatch_config.clone(); + wrong_credentials.client.secret.secret = + Some("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB".to_owned()); + let server_accept = { + let server = server.clone(); + tokio::spawn(async move { server.accept_session().await }) + }; + let wrong_client = TlsTcpConnector::with_client_config(wrong_credentials, client_tls.clone()); + assert!(matches!( + timeout(Duration::from_secs(2), wrong_client.connect()) + .await + .unwrap(), + Err(TransportError::AuthenticationRejected( + protocol::RejectReason::AuthenticationFailed + )) + )); + assert!(matches!( + timeout(Duration::from_secs(2), server_accept) + .await + .unwrap() + .unwrap(), + Err(TransportError::AuthenticationRejected( + protocol::RejectReason::AuthenticationFailed + )) + )); + + let server_accept = { + let server = server.clone(); + tokio::spawn(async move { server.accept_session().await }) + }; + let mut mismatch_config = mismatch_config; + mismatch_config.client.server = "127.0.0.1".to_owned(); + let mismatch_client = TlsTcpConnector::with_client_config(mismatch_config, client_tls); + assert!(matches!( + timeout(Duration::from_secs(2), mismatch_client.connect()) + .await + .unwrap(), + Err(TransportError::Io(_)) + )); + assert!(timeout(Duration::from_secs(2), server_accept) + .await + .unwrap() + .unwrap() + .is_err()); } diff --git a/tests/limits.rs b/tests/limits.rs new file mode 100644 index 0000000..c1bbc44 --- /dev/null +++ b/tests/limits.rs @@ -0,0 +1,141 @@ +use std::{net::SocketAddr, sync::Arc, thread, time::Duration}; + +use wgcloak::{ + config::{self, LimitsConfig}, + transport::{AdmissionController, AdmissionError}, +}; + +fn limits( + max_sessions: usize, + max_sessions_per_ip: usize, + max_pending_handshakes: usize, + max_auth_attempts_per_ip: usize, + auth_attempt_window_secs: u64, +) -> LimitsConfig { + LimitsConfig { + max_sessions, + max_sessions_per_ip, + max_pending_handshakes, + max_auth_attempts_per_ip, + auth_attempt_window_secs, + handshake_timeout_ms: 1_000, + } +} + +fn address(ip: &str) -> SocketAddr { + format!("{ip}:40000").parse().unwrap() +} + +#[test] +fn pending_limit_and_failed_handshake_release_reservation() { + let controller = AdmissionController::new(&limits(8, 4, 1, 100, 60)); + let first = controller.begin(address("192.0.2.1")).unwrap(); + assert_eq!(controller.pending_handshakes(), 1); + assert!(matches!( + controller.begin(address("192.0.2.2")), + Err(AdmissionError::Busy) + )); + + // Dropping the pending guard models any failed or timed-out handshake. + drop(first); + assert_eq!(controller.pending_handshakes(), 0); + let second = controller.begin(address("192.0.2.2")).unwrap(); + drop(second); + assert_eq!(controller.pending_handshakes(), 0); +} + +#[test] +fn active_and_per_ip_limits_release_on_close_or_drop() { + let controller = AdmissionController::new(&limits(2, 1, 2, 100, 60)); + let first = controller.begin(address("192.0.2.1")).unwrap().promote(); + assert_eq!(controller.active_sessions(), 1); + assert!(matches!( + controller.begin(address("192.0.2.1")), + Err(AdmissionError::Busy) + )); + + let second = controller.begin(address("192.0.2.2")).unwrap().promote(); + assert_eq!(controller.active_sessions(), 2); + assert!(matches!( + controller.begin(address("192.0.2.3")), + Err(AdmissionError::Busy) + )); + + drop(first); + assert_eq!(controller.active_sessions(), 1); + drop(second); + assert_eq!(controller.active_sessions(), 0); + assert!(controller.begin(address("192.0.2.3")).is_ok()); +} + +#[test] +fn concurrent_admissions_never_exceed_global_or_per_ip_capacity() { + let controller = AdmissionController::new(&limits(3, 3, 32, 100, 60)); + let barrier = Arc::new(std::sync::Barrier::new(33)); + let (sender, receiver) = std::sync::mpsc::channel(); + let mut workers = Vec::new(); + for _ in 0..32 { + let controller = controller.clone(); + let barrier = barrier.clone(); + let sender = sender.clone(); + workers.push(thread::spawn(move || { + barrier.wait(); + let permit = controller + .begin(address("192.0.2.10")) + .ok() + .map(|pending| pending.promote()); + barrier.wait(); + sender.send(permit).unwrap(); + })); + } + barrier.wait(); + barrier.wait(); + drop(sender); + let permits: Vec<_> = receiver.into_iter().flatten().collect(); + for worker in workers { + worker.join().unwrap(); + } + assert_eq!(permits.len(), 3); + assert_eq!(controller.active_sessions(), 3); +} + +#[tokio::test] +async fn authentication_attempt_limit_is_per_ip_and_expires() { + let controller = AdmissionController::new(&limits(8, 4, 8, 2, 1)); + for _ in 0..2 { + drop(controller.begin(address("192.0.2.1")).unwrap()); + } + assert!(matches!( + controller.begin(address("192.0.2.1")), + Err(AdmissionError::RateLimited) + )); + assert!(controller.begin(address("192.0.2.2")).is_ok()); + + tokio::time::sleep(Duration::from_millis(1_050)).await; + assert!(controller.begin(address("192.0.2.1")).is_ok()); +} + +#[test] +fn server_limits_have_safe_defaults_and_reject_out_of_bounds_values() { + let defaults = LimitsConfig::default(); + assert_eq!(defaults.max_sessions, 1024); + assert_eq!(defaults.max_sessions_per_ip, 32); + assert_eq!(defaults.max_pending_handshakes, 256); + assert_eq!(defaults.max_auth_attempts_per_ip, 20); + assert_eq!(defaults.auth_attempt_window_secs, 60); + assert_eq!(defaults.handshake_timeout_ms, 3_000); + + let valid = "[server]\nlisten='127.0.0.1'\n[backend]\ntarget='127.0.0.1:51820'\n[[auth.clients]]\nid='one'\nsecret='AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'\n"; + assert!(config::Config::from_str(valid).unwrap().validate().is_ok()); + + let invalid = format!("{valid}\n[limits]\nmax_auth_attempts_per_ip=0\n"); + assert!(config::Config::from_str(&invalid) + .unwrap() + .validate() + .is_err()); + let too_long = format!("{valid}\n[limits]\nhandshake_timeout_ms=300001\n"); + assert!(config::Config::from_str(&too_long) + .unwrap() + .validate() + .is_err()); +} diff --git a/tests/quic.rs b/tests/quic.rs index 8c757cf..4a10afc 100644 --- a/tests/quic.rs +++ b/tests/quic.rs @@ -1,14 +1,3 @@ -#[path = "../src/config.rs"] -mod config; -#[path = "../src/datagram.rs"] -mod datagram; -#[path = "../src/protocol/mod.rs"] -mod protocol; -#[path = "../src/stats.rs"] -mod stats; -#[path = "../src/transport/mod.rs"] -mod transport; - use std::{collections::HashMap, sync::Arc, time::Duration}; use base64::Engine; @@ -16,9 +5,12 @@ use bytes::Bytes; use quinn::{ClientConfig as QuinnClientConfig, Endpoint, ServerConfig as QuinnServerConfig}; use rustls::pki_types::{CertificateDer, PrivatePkcs8KeyDer}; use tokio::time::timeout; -use transport::{ - quic::{QuicClientConnector, QuicServer}, - TransportError, TransportSession, +use wgcloak::{ + config, + transport::{ + quic::{QuicClientConnector, QuicServer}, + AdmissionController, TransportError, TransportSession, + }, }; const SECRET: &str = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; @@ -71,12 +63,34 @@ async fn authenticated_datagram_round_trip_and_limit() { .decode(SECRET) .unwrap(), ); - let server = QuicServer::from_endpoint(endpoint, clients, 1200, Duration::from_secs(2)); - let port = server.local_addr().unwrap().port(); + let admission = AdmissionController::new(&config::LimitsConfig::default()); + let server = QuicServer::from_endpoint_with_admission( + endpoint, + clients, + 1200, + Duration::from_secs(2), + admission.clone(), + ); + let server_addr = server.local_addr().unwrap(); + let port = server_addr.port(); let client = QuicClientConnector::with_client_config(client_config(port, SECRET), client_tls); let server_task = tokio::spawn(async move { server.accept().await }); let client_session = client.connect_session().await.unwrap(); let server_session = server_task.await.unwrap().unwrap(); + let remote_addr = client_session.remote_addr().unwrap(); + assert_eq!(remote_addr.port(), server_addr.port()); + assert!(match remote_addr.ip() { + std::net::IpAddr::V4(address) => address.is_loopback(), + std::net::IpAddr::V6(address) => { + address.is_loopback() + || address + .to_ipv4_mapped() + .map(|mapped| mapped.is_loopback()) + .unwrap_or(false) + } + }); + assert!(client_session.rtt().is_some()); + assert_eq!(client_session.authenticated_keepalive_interval(), None); client_session .send_datagram(Bytes::from_static(b"opaque")) .await @@ -88,6 +102,7 @@ async fn authenticated_datagram_round_trip_and_limit() { .unwrap(), Bytes::from_static(b"opaque") ); + assert_eq!(admission.active_sessions(), 1); assert!(matches!( client_session .send_datagram(Bytes::from(vec![0; 1201])) @@ -96,6 +111,7 @@ async fn authenticated_datagram_round_trip_and_limit() { )); client_session.close().await.unwrap(); server_session.close().await.unwrap(); + assert_eq!(admission.active_sessions(), 0); } #[tokio::test] diff --git a/tests/reconnect.rs b/tests/reconnect.rs new file mode 100644 index 0000000..5172def --- /dev/null +++ b/tests/reconnect.rs @@ -0,0 +1,161 @@ +use std::{path::Path, time::Duration}; + +use tempfile::tempdir; +use tokio::{ + net::UdpSocket, + time::{sleep, timeout}, +}; +use wgcloak::{ + client, + config::{self, Config}, + control, server, +}; + +const SECRET: &str = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + +fn free_udp_port() -> u16 { + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + socket.local_addr().unwrap().port() +} + +fn server_config(port: u16, backend: std::net::SocketAddr) -> config::ServerConfig { + Config::from_str(&format!( + "[server]\nlisten='127.0.0.1'\n[backend]\ntarget='{backend}'\n[transports.direct_udp]\nenabled=true\nport={port}\n[transports.quic]\nenabled=false\n[transports.http3]\nenabled=false\n[transports.tls_tcp]\nenabled=false\n[[auth.clients]]\nid='reconnect-test'\nsecret='{SECRET}'\n" + )) + .unwrap() + .into_server() + .unwrap() +} + +fn client_config(port: u16, local_port: u16) -> config::ClientConfig { + Config::from_str(&format!( + "[client]\nserver='127.0.0.1'\nid='reconnect-test'\nsecret='{SECRET}'\n[local]\nlisten='127.0.0.1:{local_port}'\n[transport]\nmode='direct_udp'\n[transport.direct_udp]\nport={port}\n[transport.auto]\nprobe_timeout_ms=150\nretries=1\n" + )) + .unwrap() + .into_client() + .unwrap() +} + +async fn wait_for_contains(path: &Path, command: &str, needle: &str) -> String { + timeout(Duration::from_secs(10), async { + loop { + if let Ok(response) = control::send_command(path, command).await { + if response.contains(needle) { + return response; + } + } + sleep(Duration::from_millis(20)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("control {command} did not contain {needle:?}")) +} + +async fn wait_for_reconnected_stats(path: &Path) -> String { + timeout(Duration::from_secs(10), async { + loop { + if let Ok(response) = control::send_command(path, "stats").await { + if response.contains("\"state\":\"connected\"") + && response.contains("\"reconnections\":1") + { + return response; + } + } + sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("client did not publish its connected reconnect state") +} + +async fn wait_for_echo(app: &UdpSocket, local_port: u16, payload: &[u8]) { + let mut response = [0u8; 128]; + app.send_to(payload, ("127.0.0.1", local_port)) + .await + .unwrap(); + let (len, _) = timeout(Duration::from_secs(5), app.recv_from(&mut response)) + .await + .expect("application echo timed out") + .unwrap(); + assert_eq!(&response[..len], payload); +} + +#[tokio::test] +async fn client_recovers_through_backoff_after_server_restart() { + let directory = tempdir().unwrap(); + let backend = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let backend_addr = backend.local_addr().unwrap(); + let backend_task = tokio::spawn(async move { + let mut packet = vec![0u8; 2048]; + while let Ok((len, peer)) = backend.recv_from(&mut packet).await { + let _ = backend.send_to(&packet[..len], peer).await; + } + }); + + let port = free_udp_port(); + let local_port = free_udp_port(); + let runtime_dir = directory.path().join("runtime"); + let server_control = runtime_dir.join("server.sock"); + let client_control = runtime_dir.join("client.sock"); + let server_task = tokio::spawn(server::run_with_control_path( + server_config(port, backend_addr), + server_control.clone(), + )); + wait_for_contains(&server_control, "status", "transports=direct_udp").await; + + let client_task = tokio::spawn(client::run_with_http3_connector_and_control_path( + client_config(port, local_port), + client_control.clone(), + None, + )); + wait_for_contains(&client_control, "status", "connected; server=127.0.0.1").await; + let app = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + wait_for_echo(&app, local_port, b"before-restart").await; + + let stopped = control::send_command(&server_control, "shutdown") + .await + .unwrap(); + assert!(stopped.contains("\"ok\":true")); + timeout(Duration::from_secs(5), server_task) + .await + .expect("first server instance should stop") + .unwrap() + .unwrap(); + + let requested = control::send_command(&client_control, "reconnect") + .await + .unwrap(); + assert!(requested.contains("\"ok\":true")); + // Wait until the client has made a failed connection attempt and entered + // jittered backoff before starting the replacement server. This exercises + // the daemon loop without relying on a guessed sleep duration. + wait_for_contains(&client_control, "status", "error=").await; + + let replacement = tokio::spawn(server::run_with_control_path( + server_config(port, backend_addr), + server_control.clone(), + )); + wait_for_contains(&server_control, "status", "transports=direct_udp").await; + let _stats = wait_for_reconnected_stats(&client_control).await; + wait_for_echo(&app, local_port, b"after-restart").await; + + let stopped = control::send_command(&server_control, "shutdown") + .await + .unwrap(); + assert!(stopped.contains("\"ok\":true")); + timeout(Duration::from_secs(5), replacement) + .await + .expect("replacement server should stop") + .unwrap() + .unwrap(); + let stopped = control::send_command(&client_control, "shutdown") + .await + .unwrap(); + assert!(stopped.contains("\"ok\":true")); + timeout(Duration::from_secs(5), client_task) + .await + .expect("client should stop") + .unwrap() + .unwrap(); + backend_task.abort(); +} diff --git a/tests/runtime_lifecycle.rs b/tests/runtime_lifecycle.rs new file mode 100644 index 0000000..50f5b84 --- /dev/null +++ b/tests/runtime_lifecycle.rs @@ -0,0 +1,146 @@ +use std::{os::unix::fs::PermissionsExt, path::Path}; +use tempfile::tempdir; +use tokio::time::Duration; +use tokio::{net::UdpSocket, time::timeout}; +use wgcloak::{ + config::Config, + control, server, + transport::{direct_udp::DirectUdpSession, TransportSession}, +}; + +const DIRECT_SECRET: &str = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + +fn private_runtime_dir(root: &Path) -> std::path::PathBuf { + let path = root.join("runtime"); + std::fs::create_dir(&path).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap(); + path +} + +#[tokio::test] +async fn control_commands_drive_only_supported_runtime_actions() { + let dir = tempdir().unwrap(); + let path = private_runtime_dir(dir.path()).join("runtime.sock"); + let state = control::ControlState::default(); + state.enable_reconnect(); + let server = control::ControlServer::bind(&path, state.clone()) + .await + .unwrap(); + let task = tokio::spawn(server.run()); + + state.set_status("connected; transport=direct_udp").await; + state + .set_stats_json("{\"datagrams_sent\":7,\"datagrams_dropped\":1}") + .await; + let status = control::send_command(&path, "status").await.unwrap(); + assert!(status.contains("connected; transport=direct_udp")); + let stats = control::send_command(&path, "stats").await.unwrap(); + assert!(stats.contains("\"datagrams_sent\":7")); + + let reconnect = state.reconnect.notified(); + tokio::pin!(reconnect); + let response = control::send_command(&path, "reconnect").await.unwrap(); + assert!(response.contains("\"ok\":true")); + timeout(Duration::from_secs(1), &mut reconnect) + .await + .expect("reconnect should reach the running client loop"); + + let response = control::send_command(&path, "reload").await.unwrap(); + assert!(response.contains("\"ok\":false")); + assert!(response.contains("not supported")); + + let response = control::send_command(&path, "shutdown").await.unwrap(); + assert!(response.contains("\"ok\":true")); + timeout(Duration::from_secs(1), task) + .await + .expect("control server should stop on shutdown") + .unwrap() + .unwrap(); +} + +#[tokio::test] +async fn server_control_rejects_client_only_reconnect() { + let dir = tempdir().unwrap(); + let path = private_runtime_dir(dir.path()).join("server.sock"); + let state = control::ControlState::default(); + let server = control::ControlServer::bind(&path, state.clone()) + .await + .unwrap(); + let task = tokio::spawn(server.run()); + + let response = control::send_command(&path, "reconnect").await.unwrap(); + assert!(response.contains("\"ok\":false")); + assert!(response.contains("not supported")); + + state.shutdown.cancel(); + timeout(Duration::from_secs(1), task) + .await + .expect("control server should stop") + .unwrap() + .unwrap(); +} + +#[tokio::test] +async fn direct_server_control_stats_track_authenticated_session_lifecycle() { + let probe = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let port = probe.local_addr().unwrap().port(); + drop(probe); + + let config = Config::from_str(&format!( + "[server]\nlisten='127.0.0.1'\n[backend]\ntarget='127.0.0.1:9'\n[transports.direct_udp]\nport={port}\n[[auth.clients]]\nid='runtime-test'\nsecret='{DIRECT_SECRET}'\n" + )) + .unwrap() + .into_server() + .unwrap(); + let client_config = Config::from_str(&format!( + "[client]\nserver='127.0.0.1'\nid='runtime-test'\nsecret='{DIRECT_SECRET}'\n[transport.direct_udp]\nport={port}\n" + )) + .unwrap() + .into_client() + .unwrap(); + + let dir = tempdir().unwrap(); + let control_path = private_runtime_dir(dir.path()).join("direct-runtime.sock"); + let server_task = tokio::spawn(server::run_with_control_path(config, control_path.clone())); + + let session = timeout( + Duration::from_secs(3), + DirectUdpSession::connect(&client_config), + ) + .await + .expect("direct handshake should complete") + .unwrap(); + + wait_for_active_count(&control_path, 1).await; + let status = control::send_command(&control_path, "status") + .await + .unwrap(); + assert!(status.contains("active_sessions=1")); + session.close().await.unwrap(); + wait_for_active_count(&control_path, 0).await; + + let response = control::send_command(&control_path, "shutdown") + .await + .unwrap(); + assert!(response.contains("\"ok\":true")); + timeout(Duration::from_secs(3), server_task) + .await + .expect("server runtime should stop") + .unwrap() + .unwrap(); +} + +async fn wait_for_active_count(path: &std::path::Path, expected: usize) { + timeout(Duration::from_secs(4), async { + loop { + if let Ok(response) = control::send_command(path, "stats").await { + if response.contains(&format!("\"active_sessions\":{expected}")) { + return; + } + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("server control stats did not report {expected} active sessions")); +} diff --git a/tests/supervisor.rs b/tests/supervisor.rs index e379291..ae30720 100644 --- a/tests/supervisor.rs +++ b/tests/supervisor.rs @@ -1,18 +1,4 @@ -#[path = "../src/config.rs"] -mod config; -#[path = "../src/datagram.rs"] -mod datagram; -#[path = "../src/protocol/mod.rs"] -mod protocol; -#[path = "../src/stats.rs"] -mod stats; -#[path = "../src/client/supervisor.rs"] -mod supervisor; -#[path = "../src/transport/mod.rs"] -mod transport; - use bytes::Bytes; -use config::{Config, TransportMode}; use std::{ sync::{ atomic::{AtomicUsize, Ordering}, @@ -20,9 +6,13 @@ use std::{ }, time::Duration, }; -use supervisor::{ProbeFuture, TransportProbe, TransportSupervisor}; use tokio::{sync::Mutex, time::timeout}; -use transport::{TransportError, TransportFuture, TransportSession}; +use wgcloak::{ + client::supervisor::{self as supervisor, ProbeFuture, TransportProbe, TransportSupervisor}, + config::{self, Config, TransportMode}, + protocol, + transport::{TransportError, TransportFuture, TransportSession}, +}; struct Session; impl TransportSession for Session { diff --git a/tests/tls_tcp.rs b/tests/tls_tcp.rs index 3eba8bc..aeb1533 100644 --- a/tests/tls_tcp.rs +++ b/tests/tls_tcp.rs @@ -1,13 +1,11 @@ -#[path = "../src/transport/tls_tcp.rs"] -mod tls_tcp; - use std::{collections::HashMap, time::Duration}; -use tls_tcp::{ +use tokio::io::{duplex, split}; +use wgcloak::transport::tls_tcp; +use wgcloak::transport::tls_tcp::{ client_authenticate, server_authenticate, AuthChallenge, FrameCodec, FrameError, FramedReader, FramedWriter, }; -use tokio::io::{duplex, split}; #[test] fn codec_handles_split_and_coalesced_frames() {