diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index d21cec67..76e14caa 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -197,6 +197,8 @@ otherwise arbitrary — the gateway answers every name under its domain. | `EVIDENCE_SERVER` | `true` | Serve evidence files at `/evidences/` on the TLS port | | `EVIDENCE_PORT` | `80` | Internal port for evidence HTTP server | | `ALPN` | | TLS ALPN protocols (e.g. `h2,http/1.1`). Only set if backends support h2c | +| `ACCEPT_PROXY_PROTOCOL` | `false` | Read the client's address from a PROXY protocol header (v1 or v2) on every connection to `PORT`, as the dstack gateway sends when the app's `port_policy` sets `pp` for it. A connection without one is refused. See [Client address](#client-address-proxy-protocol) | +| `SEND_PROXY_PROTOCOL` | `false` | Pass the client's address to the backends (`TARGET_ENDPOINT`, `ROUTING_MAP`) with PROXY protocol v2. Only for backends that read it | | `DELEGATION_ZONE` | | Zone this container writes into, so the DNS token needs no access to the served domain's own zone (see below) | | `DELEGATION_PROPAGATION_SECONDS` | `120` | Wait after writing the delegated challenge TXT before validation. Must outlast the record TTL (60s), or a resolver still serving the previous attempt's value fails validation. The certbot run timeout is sized from this, so raising it is safe | | `CERTBOT_TIMEOUT` | propagation wait + 180s, never below 300s | Seconds a single certbot run may take. The default is derived from the provider's propagation wait (or `DELEGATION_PROPAGATION_SECONDS`), which is what dominates it. Set this only if a run needs longer still; an explicit value is used as given, floor included | @@ -275,6 +277,28 @@ start the container and create the records afterwards; `print` lists them and continues without checking; `webhook` POSTs them to `DNS_WEBHOOK_URL` for an operator service to create automatically. +## Client address (PROXY protocol) + +The gateway relays TCP, so haproxy sees each connection come from the gateway, and the backend sees it come from +haproxy. To give the backend the client's address instead (requires dstack-gateway 0.5.9 or later): + +1. Have the gateway send it: in the app compose, set `"port_policy": {"ports": [{"port": , "pp": true}]}`, + with `PORT` as configured for this container (on Phala Cloud, the port policy's "Proxy protocol (pp)" switch). + Guest OS 0.5.9+ reports the policy when the CVM registers with the gateway; older images need `public_tcbinfo` + on so the gateway can read it from the TCB info. +2. Set `ACCEPT_PROXY_PROTOCOL=true`, so haproxy reads that header. Change both in the same deployment: a header + haproxy does not expect, or one it expects and does not get, fails every connection. This includes anything + that reaches `PORT` without the gateway, such as a host port mapping or another container in the CVM. +3. Set `SEND_PROXY_PROTOCOL=true`, and have the backend accept PROXY protocol v2 from haproxy only, for example + Caddy's `proxy_protocol` listener wrapper or nginx's `listen ... proxy_protocol` with `set_real_ip_from`. With + `ROUTING_MAP`, this applies to every backend. + +In tls-alpn-01 mode the address also survives the hop to the TLS frontend on loopback, which already uses the PROXY +protocol internally. The ACME responder never sees the header. + +If the gateway itself sits behind an L4 load balancer, the gateway operator must enable `inbound_pp_enabled` on it +(and PROXY protocol on the load balancer), or the address passed on is the load balancer's. + ## Evidence & Attestation Evidence files are served at `https://your-domain.com/evidences/` by default (via payload inspection in HAProxy's TCP mode). They can also be accessed by the backend application through the shared `/evidences` volume. diff --git a/custom-domain/dstack-ingress/TESTING.md b/custom-domain/dstack-ingress/TESTING.md index d4a9b6d2..cf16b88c 100644 --- a/custom-domain/dstack-ingress/TESTING.md +++ b/custom-domain/dstack-ingress/TESTING.md @@ -335,6 +335,12 @@ python3 scripts/tests/test_dnsguide.py # DNS/CAA parsing and record building bash scripts/tests/test_sanitizers.sh # env var validation ``` +Against a built image: + +```bash +bash scripts/tests/test_proxy_protocol.sh # ACCEPT_/SEND_PROXY_PROTOCOL end to end +``` + The parsing tests are built from rdata real resolvers actually returned, in both encodings. When you touch DNS handling, add the real string you saw rather than one you composed — the encodings are the part that surprises people. diff --git a/custom-domain/dstack-ingress/scripts/dns01.sh b/custom-domain/dstack-ingress/scripts/dns01.sh index 91e409e8..30241661 100644 --- a/custom-domain/dstack-ingress/scripts/dns01.sh +++ b/custom-domain/dstack-ingress/scripts/dns01.sh @@ -407,7 +407,7 @@ run_pass build_combined_pems haproxy_emit_global -haproxy_emit_tls_frontend ":${PORT}" +haproxy_emit_tls_frontend ":${PORT}$(haproxy_accept_proxy)" haproxy_emit_backends evidence_start_server diff --git a/custom-domain/dstack-ingress/scripts/entrypoint.sh b/custom-domain/dstack-ingress/scripts/entrypoint.sh index 2656c198..ab226139 100644 --- a/custom-domain/dstack-ingress/scripts/entrypoint.sh +++ b/custom-domain/dstack-ingress/scripts/entrypoint.sh @@ -30,6 +30,8 @@ TIMEOUT_SERVER=${TIMEOUT_SERVER:-86400s} EVIDENCE_SERVER=${EVIDENCE_SERVER:-true} EVIDENCE_PORT=${EVIDENCE_PORT:-80} ALPN=${ALPN:-} +ACCEPT_PROXY_PROTOCOL=${ACCEPT_PROXY_PROTOCOL:-false} +SEND_PROXY_PROTOCOL=${SEND_PROXY_PROTOCOL:-false} CHALLENGE_TYPE=${CHALLENGE_TYPE:-dns-01} # ACME account settings, normalised once for both modes. What you are @@ -83,6 +85,12 @@ fi if ! ALPN=$(sanitize_alpn "$ALPN"); then exit 1 fi +if ! ACCEPT_PROXY_PROTOCOL=$(sanitize_boolean "$ACCEPT_PROXY_PROTOCOL" "ACCEPT_PROXY_PROTOCOL"); then + exit 1 +fi +if ! SEND_PROXY_PROTOCOL=$(sanitize_boolean "$SEND_PROXY_PROTOCOL" "SEND_PROXY_PROTOCOL"); then + exit 1 +fi # Warn about deprecated L7 env vars for var in CLIENT_MAX_BODY_SIZE PROXY_READ_TIMEOUT PROXY_SEND_TIMEOUT PROXY_CONNECT_TIMEOUT PROXY_BUFFER_SIZE PROXY_BUFFERS PROXY_BUSY_BUFFERS_SIZE; do @@ -90,11 +98,15 @@ for var in CLIENT_MAX_BODY_SIZE PROXY_READ_TIMEOUT PROXY_SEND_TIMEOUT PROXY_CONN echo "Warning: $var is ignored in TCP proxy mode" fi done +if [ "$SEND_PROXY_PROTOCOL" = "true" ] && [ "$ACCEPT_PROXY_PROTOCOL" != "true" ]; then + echo "Warning: SEND_PROXY_PROTOCOL without ACCEPT_PROXY_PROTOCOL passes the gateway's address, not the client's" +fi # Everything from here on belongs to one mode. Exported so the mode script and # the helpers it invokes see the sanitized values. export PORT DOMAIN DOMAINS TARGET_ENDPOINT ROUTING_MAP TXT_PREFIX MAXCONN export TIMEOUT_CONNECT TIMEOUT_CLIENT TIMEOUT_SERVER EVIDENCE_SERVER EVIDENCE_PORT ALPN +export ACCEPT_PROXY_PROTOCOL SEND_PROXY_PROTOCOL export ACME_EMAIL ACME_STAGING case "$CHALLENGE_TYPE" in diff --git a/custom-domain/dstack-ingress/scripts/functions.sh b/custom-domain/dstack-ingress/scripts/functions.sh index 17dc396f..d9748c7f 100644 --- a/custom-domain/dstack-ingress/scripts/functions.sh +++ b/custom-domain/dstack-ingress/scripts/functions.sh @@ -111,6 +111,18 @@ sanitize_alpn() { fi } +sanitize_boolean() { + local candidate="$1" + local name="$2" + case "$candidate" in + true|false) echo "$candidate" ;; + *) + echo "Error: $name must be true or false: $candidate" >&2 + return 1 + ;; + esac +} + sanitize_proxy_timeout() { local candidate="$1" if [ -z "$candidate" ]; then diff --git a/custom-domain/dstack-ingress/scripts/haproxy-lib.sh b/custom-domain/dstack-ingress/scripts/haproxy-lib.sh index dcaf4b82..ec84f805 100644 --- a/custom-domain/dstack-ingress/scripts/haproxy-lib.sh +++ b/custom-domain/dstack-ingress/scripts/haproxy-lib.sh @@ -44,6 +44,15 @@ defaults EOF } +# PROXY protocol, opt-in at each end: accept it from the gateway on the public +# bind, send it (v2) to the app backends. +haproxy_accept_proxy() { + if [ "${ACCEPT_PROXY_PROTOCOL:-false}" = "true" ]; then printf ' accept-proxy'; fi +} +haproxy_send_proxy() { + if [ "${SEND_PROXY_PROTOCOL:-false}" = "true" ]; then printf ' send-proxy-v2'; fi +} + # The TLS-terminating frontend. $1 is the bind spec -- the one part the modes # disagree about. haproxy_emit_tls_frontend() { @@ -97,7 +106,7 @@ haproxy_emit_backends_single() { default_backend be_upstream backend be_upstream - server app1 ${target_hostport} + server app1 ${target_hostport}$(haproxy_send_proxy) EOF } @@ -147,7 +156,7 @@ haproxy_emit_backends_multi() { backend_sections="${backend_sections} backend ${be_name} - server s1 ${target}" + server s1 ${target}$(haproxy_send_proxy)" done <<< "$routing_map_normalized" echo "$backend_rules" >> /etc/haproxy/haproxy.cfg diff --git a/custom-domain/dstack-ingress/scripts/tests/test_proxy_protocol.sh b/custom-domain/dstack-ingress/scripts/tests/test_proxy_protocol.sh new file mode 100755 index 00000000..607a825a --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/tests/test_proxy_protocol.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# Verify the opt-in PROXY protocol: with ACCEPT_PROXY_PROTOCOL and +# SEND_PROXY_PROTOCOL, the public bind takes the client's address from the +# header the dstack gateway prepends, refuses a connection without one, and +# hands that address to the backend; with neither, the config is unchanged. +# +# Usage: ./scripts/tests/test_proxy_protocol.sh + +set -euo pipefail + +IMAGE="${1:?usage: $0 }" +CONTAINER="dstack-ingress-proxy-protocol-$$" + +# shellcheck disable=SC2317 # Invoked by the EXIT trap below. +cleanup() { + docker rm -f "${CONTAINER}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +failures=0 +check() { + local msg="$1" expected="$2" actual="$3" + if [[ "$actual" == "$expected" ]]; then + echo "PASS: ${msg}" + else + echo "FAIL: ${msg} (expected '${expected}', got '${actual}')" >&2 + failures=$((failures + 1)) + fi +} + +# The config the helpers emit with the switches unset has no PROXY keyword. +default_cfg="$(docker run --rm --entrypoint bash "${IMAGE}" -c ' + set -euo pipefail + source /scripts/functions.sh + source /scripts/haproxy-lib.sh + MAXCONN=16 TIMEOUT_CONNECT=5s TIMEOUT_CLIENT=30s TIMEOUT_SERVER=30s + EVIDENCE_SERVER=false DOMAIN=localhost TARGET_ENDPOINT=127.0.0.1:25080 + haproxy_emit_global + haproxy_emit_tls_frontend "127.0.0.1:24443$(haproxy_accept_proxy)" + haproxy_emit_backends + cat /etc/haproxy/haproxy.cfg +')" +check "default config has no accept-proxy" "0" "$(grep -c 'accept-proxy' <<<"${default_cfg}" || true)" +check "default config has no send-proxy" "0" "$(grep -c 'send-proxy' <<<"${default_cfg}" || true)" + +docker run -d --rm --name "${CONTAINER}" --entrypoint bash "${IMAGE}" -c ' + set -euo pipefail + mkdir -p /etc/haproxy/certs + openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout /tmp/key.pem -out /tmp/cert.pem \ + -subj /CN=localhost -days 1 >/dev/null 2>&1 + cat /tmp/key.pem /tmp/cert.pem >/etc/haproxy/certs/test.pem + + # The backend reports the source address of the PROXY v2 header it receives. + python3 - <<"PY" & +import socket, struct +srv = socket.socket(); srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +srv.bind(("127.0.0.1", 25080)); srv.listen() +def recv_exact(conn, n): + buf = b"" + while len(buf) < n and (chunk := conn.recv(n - len(buf))): + buf += chunk + return buf +while True: + conn, _ = srv.accept() + head = recv_exact(conn, 16) + if head[:12] != b"\r\n\r\n\x00\r\nQUIT\n": + conn.sendall(b"src=none"); conn.close(); continue + body = recv_exact(conn, struct.unpack("!H", head[14:16])[0]) + conn.sendall(("src=" + socket.inet_ntoa(body[:4])).encode()); conn.close() +PY + + source /scripts/functions.sh + source /scripts/haproxy-lib.sh + MAXCONN=16 TIMEOUT_CONNECT=5s TIMEOUT_CLIENT=30s TIMEOUT_SERVER=30s + EVIDENCE_SERVER=false DOMAIN=localhost TARGET_ENDPOINT=127.0.0.1:25080 + ACCEPT_PROXY_PROTOCOL=true SEND_PROXY_PROTOCOL=true + haproxy_emit_global + haproxy_emit_tls_frontend "127.0.0.1:24443$(haproxy_accept_proxy)" + haproxy_emit_backends + exec haproxy -W -db -f /etc/haproxy/haproxy.cfg +' >/dev/null + +client=' +import socket, ssl, struct, sys +def pp2(src): + addr = socket.inet_aton(src) + socket.inet_aton("127.0.0.1") + struct.pack("!HH", 40000, 24443) + return b"\r\n\r\n\x00\r\nQUIT\n" + bytes([0x21, 0x11]) + struct.pack("!H", len(addr)) + addr +ctx = ssl.create_default_context(); ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE +raw = socket.create_connection(("127.0.0.1", 24443), timeout=5) +if sys.argv[1] != "none": + raw.sendall(pp2(sys.argv[1])) +try: + tls = ctx.wrap_socket(raw, server_hostname="localhost") + tls.sendall(b"x") + print(tls.recv(64).decode()) +except (ssl.SSLError, ConnectionError, OSError): + print("refused") +' + +result="" +for _ in {1..50}; do + result="$(docker exec "${CONTAINER}" python3 -c "${client}" 203.0.113.9 2>/dev/null || true)" + [[ -n "$result" && "$result" != "refused" ]] && break + sleep 0.1 +done +check "backend receives the client address from the PROXY header" "src=203.0.113.9" "${result}" +check "a connection without a PROXY header is refused" "refused" \ + "$(docker exec "${CONTAINER}" python3 -c "${client}" none 2>/dev/null || true)" + +if [[ $failures -eq 0 ]]; then + echo "All PROXY protocol tests passed" +else + echo "$failures PROXY protocol tests failed" >&2 + exit 1 +fi diff --git a/custom-domain/dstack-ingress/scripts/tests/test_sanitizers.sh b/custom-domain/dstack-ingress/scripts/tests/test_sanitizers.sh index 5c1bda4a..fccf04aa 100644 --- a/custom-domain/dstack-ingress/scripts/tests/test_sanitizers.sh +++ b/custom-domain/dstack-ingress/scripts/tests/test_sanitizers.sh @@ -67,6 +67,8 @@ assert_equal "$(sanitize_alpn 'h2,http/1.1')" "h2,http/1.1" "sanitize_alpn accep assert_equal "$(sanitize_alpn 'h2')" "h2" "sanitize_alpn accepts h2" assert_equal "$(sanitize_alpn 'http/1.1')" "http/1.1" "sanitize_alpn accepts http/1.1" assert_equal "$(sanitize_alpn '')" "" "sanitize_alpn accepts empty" +assert_equal "$(sanitize_boolean true FLAG)" "true" "sanitize_boolean accepts true" +assert_equal "$(sanitize_boolean false FLAG)" "false" "sanitize_boolean accepts false" # Failing cases assert_fails "sanitize_port rejects non-numeric" sanitize_port abc @@ -108,6 +110,8 @@ assert_fails "sanitize_haproxy_timeout rejects bare number" sanitize_haproxy_tim assert_fails "sanitize_alpn rejects semicolons" sanitize_alpn "h2;drop" assert_fails "sanitize_alpn rejects newlines" sanitize_alpn $'h2\nhttp/1.1' assert_fails "sanitize_alpn rejects spaces" sanitize_alpn "h2, http/1.1" +assert_fails "sanitize_boolean rejects yes" sanitize_boolean yes FLAG +assert_fails "sanitize_boolean rejects empty" sanitize_boolean "" FLAG if [[ $failures -eq 0 ]]; then echo "All sanitizer tests passed" diff --git a/custom-domain/dstack-ingress/scripts/tlsalpn.sh b/custom-domain/dstack-ingress/scripts/tlsalpn.sh index 05d0b6d5..b42f7958 100644 --- a/custom-domain/dstack-ingress/scripts/tlsalpn.sh +++ b/custom-domain/dstack-ingress/scripts/tlsalpn.sh @@ -121,7 +121,7 @@ txt_record_value() { echo "${DSTACK_INSTANCE_ID}:${PORT}"; } emit_peek_frontend() { cat <>/etc/haproxy/haproxy.cfg frontend tls_peek - bind :${PORT} + bind :${PORT}$(haproxy_accept_proxy) tcp-request inspect-delay 5s tcp-request content accept if { req.ssl_hello_type 1 } # Only the CA sends this ALPN protocol. A client that sends it anyway just