feat(ingress): opt-in PROXY protocol for the client address

custom-domain/dstack-ingress/README.md

@@ -197,6 +197,8 @@ otherwise arbitrary — the gateway answers every name under its domain.
 | `EVIDENCE_SERVER` | `true` | Serve evidence files at `/evidences/` on the TLS port |
 | `EVIDENCE_PORT` | `80` | Internal port for evidence HTTP server |
 | `ALPN` | | TLS ALPN protocols (e.g. `h2,http/1.1`). Only set if backends support h2c |
+| `ACCEPT_PROXY_PROTOCOL` | `false` | Read the client's address from a PROXY protocol header (v1 or v2) on every connection to `PORT`, as the dstack gateway sends when the app's `port_policy` sets `pp` for it. A connection without one is refused. See [Client address](#client-address-proxy-protocol) |
+| `SEND_PROXY_PROTOCOL` | `false` | Pass the client's address to the backends (`TARGET_ENDPOINT`, `ROUTING_MAP`) with PROXY protocol v2. Only for backends that read it |
 | `DELEGATION_ZONE` | | Zone this container writes into, so the DNS token needs no access to the served domain's own zone (see below) |
 | `DELEGATION_PROPAGATION_SECONDS` | `120` | Wait after writing the delegated challenge TXT before validation. Must outlast the record TTL (60s), or a resolver still serving the previous attempt's value fails validation. The certbot run timeout is sized from this, so raising it is safe |
 | `CERTBOT_TIMEOUT` | propagation wait + 180s, never below 300s | Seconds a single certbot run may take. The default is derived from the provider's propagation wait (or `DELEGATION_PROPAGATION_SECONDS`), which is what dominates it. Set this only if a run needs longer still; an explicit value is used as given, floor included |
@@ -275,6 +277,28 @@ start the container and create the records afterwards; `print` lists them and
 continues without checking; `webhook` POSTs them to `DNS_WEBHOOK_URL` for an
 operator service to create automatically.
 
+## Client address (PROXY protocol)
+
+The gateway relays TCP, so haproxy sees each connection come from the gateway, and the backend sees it come from
+haproxy. To give the backend the client's address instead (requires dstack-gateway 0.5.9 or later):
+
+1. Have the gateway send it: in the app compose, set `"port_policy": {"ports": [{"port": <PORT>, "pp": true}]}`,
+   with `PORT` as configured for this container (on Phala Cloud, the port policy's "Proxy protocol (pp)" switch).
+   Guest OS 0.5.9+ reports the policy when the CVM registers with the gateway; older images need `public_tcbinfo`
+   on so the gateway can read it from the TCB info.
+2. Set `ACCEPT_PROXY_PROTOCOL=true`, so haproxy reads that header. Change both in the same deployment: a header
+   haproxy does not expect, or one it expects and does not get, fails every connection. This includes anything
+   that reaches `PORT` without the gateway, such as a host port mapping or another container in the CVM.
+3. Set `SEND_PROXY_PROTOCOL=true`, and have the backend accept PROXY protocol v2 from haproxy only, for example
+   Caddy's `proxy_protocol` listener wrapper or nginx's `listen ... proxy_protocol` with `set_real_ip_from`. With
+   `ROUTING_MAP`, this applies to every backend.
+
+In tls-alpn-01 mode the address also survives the hop to the TLS frontend on loopback, which already uses the PROXY
+protocol internally. The ACME responder never sees the header.
+
+If the gateway itself sits behind an L4 load balancer, the gateway operator must enable `inbound_pp_enabled` on it
+(and PROXY protocol on the load balancer), or the address passed on is the load balancer's.
+
 ## Evidence & Attestation
 
 Evidence files are served at `https://your-domain.com/evidences/` by default (via payload inspection in HAProxy's TCP mode). They can also be accessed by the backend application through the shared `/evidences` volume.

custom-domain/dstack-ingress/TESTING.md

@@ -335,6 +335,12 @@ python3 scripts/tests/test_dnsguide.py     # DNS/CAA parsing and record building
 bash    scripts/tests/test_sanitizers.sh   # env var validation
 ```
 
+Against a built image:
+
+```bash
+bash scripts/tests/test_proxy_protocol.sh <image>     # ACCEPT_/SEND_PROXY_PROTOCOL end to end
+```
+
 The parsing tests are built from rdata real resolvers actually returned, in both
 encodings. When you touch DNS handling, add the real string you saw rather than
 one you composed — the encodings are the part that surprises people.

custom-domain/dstack-ingress/scripts/dns01.sh

@@ -407,7 +407,7 @@ run_pass
 build_combined_pems
 
 haproxy_emit_global
-haproxy_emit_tls_frontend ":${PORT}"
+haproxy_emit_tls_frontend ":${PORT}$(haproxy_accept_proxy)"
 haproxy_emit_backends
 
 evidence_start_server

custom-domain/dstack-ingress/scripts/entrypoint.sh

@@ -30,6 +30,8 @@ TIMEOUT_SERVER=${TIMEOUT_SERVER:-86400s}
 EVIDENCE_SERVER=${EVIDENCE_SERVER:-true}
 EVIDENCE_PORT=${EVIDENCE_PORT:-80}
 ALPN=${ALPN:-}
+ACCEPT_PROXY_PROTOCOL=${ACCEPT_PROXY_PROTOCOL:-false}
+SEND_PROXY_PROTOCOL=${SEND_PROXY_PROTOCOL:-false}
 CHALLENGE_TYPE=${CHALLENGE_TYPE:-dns-01}
 
 # ACME account settings, normalised once for both modes. What you are
@@ -83,18 +85,28 @@ fi
 if ! ALPN=$(sanitize_alpn "$ALPN"); then
     exit 1
 fi
+if ! ACCEPT_PROXY_PROTOCOL=$(sanitize_boolean "$ACCEPT_PROXY_PROTOCOL" "ACCEPT_PROXY_PROTOCOL"); then
+    exit 1
+fi
+if ! SEND_PROXY_PROTOCOL=$(sanitize_boolean "$SEND_PROXY_PROTOCOL" "SEND_PROXY_PROTOCOL"); then
+    exit 1
+fi
 
 # Warn about deprecated L7 env vars
 for var in CLIENT_MAX_BODY_SIZE PROXY_READ_TIMEOUT PROXY_SEND_TIMEOUT PROXY_CONNECT_TIMEOUT PROXY_BUFFER_SIZE PROXY_BUFFERS PROXY_BUSY_BUFFERS_SIZE; do
     if [ -n "${!var}" ]; then
         echo "Warning: $var is ignored in TCP proxy mode"
     fi
 done
+if [ "$SEND_PROXY_PROTOCOL" = "true" ] && [ "$ACCEPT_PROXY_PROTOCOL" != "true" ]; then
+    echo "Warning: SEND_PROXY_PROTOCOL without ACCEPT_PROXY_PROTOCOL passes the gateway's address, not the client's"
+fi
 
 # Everything from here on belongs to one mode. Exported so the mode script and
 # the helpers it invokes see the sanitized values.
 export PORT DOMAIN DOMAINS TARGET_ENDPOINT ROUTING_MAP TXT_PREFIX MAXCONN
 export TIMEOUT_CONNECT TIMEOUT_CLIENT TIMEOUT_SERVER EVIDENCE_SERVER EVIDENCE_PORT ALPN
+export ACCEPT_PROXY_PROTOCOL SEND_PROXY_PROTOCOL
 export ACME_EMAIL ACME_STAGING
 
 case "$CHALLENGE_TYPE" in

custom-domain/dstack-ingress/scripts/functions.sh

@@ -111,6 +111,18 @@ sanitize_alpn() {
     fi
 }
 
+sanitize_boolean() {
+    local candidate="$1"
+    local name="$2"
+    case "$candidate" in
+        true|false) echo "$candidate" ;;
+        *)
+            echo "Error: $name must be true or false: $candidate" >&2
+            return 1
+            ;;
+    esac
+}
+
 sanitize_proxy_timeout() {
     local candidate="$1"
     if [ -z "$candidate" ]; then

custom-domain/dstack-ingress/scripts/haproxy-lib.sh

@@ -44,6 +44,15 @@ defaults
 EOF
 }
 
+# PROXY protocol, opt-in at each end: accept it from the gateway on the public
+# bind, send it (v2) to the app backends.
+haproxy_accept_proxy() {
+    if [ "${ACCEPT_PROXY_PROTOCOL:-false}" = "true" ]; then printf ' accept-proxy'; fi
+}
+haproxy_send_proxy() {
+    if [ "${SEND_PROXY_PROTOCOL:-false}" = "true" ]; then printf ' send-proxy-v2'; fi
+}
+
 # The TLS-terminating frontend. $1 is the bind spec -- the one part the modes
 # disagree about.
 haproxy_emit_tls_frontend() {
@@ -97,7 +106,7 @@ haproxy_emit_backends_single() {
     default_backend be_upstream
 
 backend be_upstream
-    server app1 ${target_hostport}
+    server app1 ${target_hostport}$(haproxy_send_proxy)
 EOF
 }
 
@@ -147,7 +156,7 @@ haproxy_emit_backends_multi() {
         backend_sections="${backend_sections}
 
 backend ${be_name}
-    server s1 ${target}"
+    server s1 ${target}$(haproxy_send_proxy)"
     done <<< "$routing_map_normalized"
 
     echo "$backend_rules" >> /etc/haproxy/haproxy.cfg

custom-domain/dstack-ingress/scripts/tests/test_proxy_protocol.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+# Verify the opt-in PROXY protocol: with ACCEPT_PROXY_PROTOCOL and
+# SEND_PROXY_PROTOCOL, the public bind takes the client's address from the
+# header the dstack gateway prepends, refuses a connection without one, and
+# hands that address to the backend; with neither, the config is unchanged.
+#
+# Usage: ./scripts/tests/test_proxy_protocol.sh <image>
+
+set -euo pipefail
+
+IMAGE="${1:?usage: $0 <image>}"
+CONTAINER="dstack-ingress-proxy-protocol-$$"
+
+# shellcheck disable=SC2317 # Invoked by the EXIT trap below.
+cleanup() {
+    docker rm -f "${CONTAINER}" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+failures=0
+check() {
+    local msg="$1" expected="$2" actual="$3"
+    if [[ "$actual" == "$expected" ]]; then
+        echo "PASS: ${msg}"
+    else
+        echo "FAIL: ${msg} (expected '${expected}', got '${actual}')" >&2
+        failures=$((failures + 1))
+    fi
+}
+
+# The config the helpers emit with the switches unset has no PROXY keyword.
+default_cfg="$(docker run --rm --entrypoint bash "${IMAGE}" -c '
+    set -euo pipefail
+    source /scripts/functions.sh
+    source /scripts/haproxy-lib.sh
+    MAXCONN=16 TIMEOUT_CONNECT=5s TIMEOUT_CLIENT=30s TIMEOUT_SERVER=30s
+    EVIDENCE_SERVER=false DOMAIN=localhost TARGET_ENDPOINT=127.0.0.1:25080
+    haproxy_emit_global
+    haproxy_emit_tls_frontend "127.0.0.1:24443$(haproxy_accept_proxy)"
+    haproxy_emit_backends
+    cat /etc/haproxy/haproxy.cfg
+')"
+check "default config has no accept-proxy" "0" "$(grep -c 'accept-proxy' <<<"${default_cfg}" || true)"
+check "default config has no send-proxy" "0" "$(grep -c 'send-proxy' <<<"${default_cfg}" || true)"
+
+docker run -d --rm --name "${CONTAINER}" --entrypoint bash "${IMAGE}" -c '
+    set -euo pipefail
+    mkdir -p /etc/haproxy/certs
+    openssl req -x509 -newkey rsa:2048 -nodes \
+        -keyout /tmp/key.pem -out /tmp/cert.pem \
+        -subj /CN=localhost -days 1 >/dev/null 2>&1
+    cat /tmp/key.pem /tmp/cert.pem >/etc/haproxy/certs/test.pem
+
+    # The backend reports the source address of the PROXY v2 header it receives.
+    python3 - <<"PY" &
+import socket, struct
+srv = socket.socket(); srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+srv.bind(("127.0.0.1", 25080)); srv.listen()
+def recv_exact(conn, n):
+    buf = b""
+    while len(buf) < n and (chunk := conn.recv(n - len(buf))):
+        buf += chunk
+    return buf
+while True:
+    conn, _ = srv.accept()
+    head = recv_exact(conn, 16)
+    if head[:12] != b"\r\n\r\n\x00\r\nQUIT\n":
+        conn.sendall(b"src=none"); conn.close(); continue
+    body = recv_exact(conn, struct.unpack("!H", head[14:16])[0])
+    conn.sendall(("src=" + socket.inet_ntoa(body[:4])).encode()); conn.close()
+PY
+
+    source /scripts/functions.sh
+    source /scripts/haproxy-lib.sh
+    MAXCONN=16 TIMEOUT_CONNECT=5s TIMEOUT_CLIENT=30s TIMEOUT_SERVER=30s
+    EVIDENCE_SERVER=false DOMAIN=localhost TARGET_ENDPOINT=127.0.0.1:25080
+    ACCEPT_PROXY_PROTOCOL=true SEND_PROXY_PROTOCOL=true
+    haproxy_emit_global
+    haproxy_emit_tls_frontend "127.0.0.1:24443$(haproxy_accept_proxy)"
+    haproxy_emit_backends
+    exec haproxy -W -db -f /etc/haproxy/haproxy.cfg
+' >/dev/null
+
+client='
+import socket, ssl, struct, sys
+def pp2(src):
+    addr = socket.inet_aton(src) + socket.inet_aton("127.0.0.1") + struct.pack("!HH", 40000, 24443)
+    return b"\r\n\r\n\x00\r\nQUIT\n" + bytes([0x21, 0x11]) + struct.pack("!H", len(addr)) + addr
+ctx = ssl.create_default_context(); ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE
+raw = socket.create_connection(("127.0.0.1", 24443), timeout=5)
+if sys.argv[1] != "none":
+    raw.sendall(pp2(sys.argv[1]))
+try:
+    tls = ctx.wrap_socket(raw, server_hostname="localhost")
+    tls.sendall(b"x")
+    print(tls.recv(64).decode())
+except (ssl.SSLError, ConnectionError, OSError):
+    print("refused")
+'
+
+result=""
+for _ in {1..50}; do
+    result="$(docker exec "${CONTAINER}" python3 -c "${client}" 203.0.113.9 2>/dev/null || true)"
+    [[ -n "$result" && "$result" != "refused" ]] && break
+    sleep 0.1
+done
+check "backend receives the client address from the PROXY header" "src=203.0.113.9" "${result}"
+check "a connection without a PROXY header is refused" "refused" \
+    "$(docker exec "${CONTAINER}" python3 -c "${client}" none 2>/dev/null || true)"
+
+if [[ $failures -eq 0 ]]; then
+    echo "All PROXY protocol tests passed"
+else
+    echo "$failures PROXY protocol tests failed" >&2
+    exit 1
+fi

custom-domain/dstack-ingress/scripts/tests/test_sanitizers.sh

@@ -67,6 +67,8 @@ assert_equal "$(sanitize_alpn 'h2,http/1.1')" "h2,http/1.1" "sanitize_alpn accep
 assert_equal "$(sanitize_alpn 'h2')" "h2" "sanitize_alpn accepts h2"
 assert_equal "$(sanitize_alpn 'http/1.1')" "http/1.1" "sanitize_alpn accepts http/1.1"
 assert_equal "$(sanitize_alpn '')" "" "sanitize_alpn accepts empty"
+assert_equal "$(sanitize_boolean true FLAG)" "true" "sanitize_boolean accepts true"
+assert_equal "$(sanitize_boolean false FLAG)" "false" "sanitize_boolean accepts false"
 
 # Failing cases
 assert_fails "sanitize_port rejects non-numeric" sanitize_port abc
@@ -108,6 +110,8 @@ assert_fails "sanitize_haproxy_timeout rejects bare number" sanitize_haproxy_tim
 assert_fails "sanitize_alpn rejects semicolons" sanitize_alpn "h2;drop"
 assert_fails "sanitize_alpn rejects newlines" sanitize_alpn $'h2\nhttp/1.1'
 assert_fails "sanitize_alpn rejects spaces" sanitize_alpn "h2, http/1.1"
+assert_fails "sanitize_boolean rejects yes" sanitize_boolean yes FLAG
+assert_fails "sanitize_boolean rejects empty" sanitize_boolean "" FLAG
 
 if [[ $failures -eq 0 ]]; then
     echo "All sanitizer tests passed"

custom-domain/dstack-ingress/scripts/tlsalpn.sh

@@ -121,7 +121,7 @@ txt_record_value() { echo "${DSTACK_INSTANCE_ID}:${PORT}"; }
 emit_peek_frontend() {
     cat <<EOF >>/etc/haproxy/haproxy.cfg
 frontend tls_peek
-    bind :${PORT}
+    bind :${PORT}$(haproxy_accept_proxy)
     tcp-request inspect-delay 5s
     tcp-request content accept if { req.ssl_hello_type 1 }
     # Only the CA sends this ALPN protocol. A client that sends it anyway just