From 5e39451b7d03e66b93c171f5e392c18c0eb2bcc1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 14 Oct 2026 04:14:55 -0700 Subject: [PATCH 1/2] fix(ingress): keep serving existing certificates when a pass fails Startup exited on any failed first pass, even with valid certificates on disk, so a failing renewal, an unreachable CA or a GetKey error turned into a crash loop that hit the CA on every restart. Exit only when some domain has nothing to serve; otherwise serve and let the loop retry. run_pass now also detects a changed certificate by its checksum, so a certificate that reached disk before a later step failed (the delegated CAA write) is still applied instead of being found current and never served. Signed-off-by: Kevin Wang --- custom-domain/dstack-ingress/scripts/dns01.sh | 31 ++++++++++++++++--- .../dstack-ingress/scripts/tlsalpn.sh | 10 +++++- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/custom-domain/dstack-ingress/scripts/dns01.sh b/custom-domain/dstack-ingress/scripts/dns01.sh index 11ee5e1..8258aa9 100644 --- a/custom-domain/dstack-ingress/scripts/dns01.sh +++ b/custom-domain/dstack-ingress/scripts/dns01.sh @@ -332,16 +332,24 @@ FIRST_PASS=true # published. APPLY_PENDING=true +# Empty when the domain has no certificate yet. +cert_checksum() { sha256sum "$(cert_fullchain_path "$1")" 2>/dev/null || true; } + # One pass over every domain: make the DNS records right, then issue or renew. # Idempotent, so the first pass and every later one are the same code. run_pass() { - local domain status failed=0 changed=0 + local domain status failed=0 changed=0 before while IFS= read -r domain; do [[ -n "$domain" ]] || continue + before=$(cert_checksum "$domain") # `if` context, not a bare command: process_domain returns 2 for # "nothing to do", and under `set -e` a bare non-zero would kill the # whole script on every quiet renewal pass. if ( process_domain "$domain" ); then status=0; else status=$?; fi + # A step after issuance can still fail (a CAA write, a timeout). The + # new certificate is on disk by then and the next pass will find it + # current, so apply it now or never. + [ "$(cert_checksum "$domain")" = "$before" ] || changed=1 case $status in 0) changed=1 ;; 2) ;; @@ -403,14 +411,29 @@ setup_certbot_env load_dstack_identity # dns-01 needs no inbound traffic, so keep the original order: certificate -# first, serving second. A certificate failure here is fatal, as it always -# was; an apply failure is left to the loop to retry. +# first, serving second. A certificate failure here is fatal only when there is +# nothing to serve. With certificates from an earlier run on disk, exiting +# would just restart into the same failure -- hitting the CA again on every +# restart -- while the certificates go unserved; serve them and let the loop +# retry with backoff instead. +have_all_certs() { + local domain + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + [ -f "$(cert_fullchain_path "$domain")" ] && [ -f "$(cert_privkey_path "$domain")" ] || return 1 + done <<<"$(get-all-domains.sh)" +} + startup_status=0 run_pass || startup_status=$? case $startup_status in 0) next_check=${RENEW_INTERVAL:-43200} ;; 3) next_check=60 ;; - *) exit "$startup_status" ;; + *) + have_all_certs || exit "$startup_status" + echo "Certificate pass failed; serving the existing certificates and retrying" >&2 + next_check=60 + ;; esac build_combined_pems diff --git a/custom-domain/dstack-ingress/scripts/tlsalpn.sh b/custom-domain/dstack-ingress/scripts/tlsalpn.sh index a43a874..da272ea 100644 --- a/custom-domain/dstack-ingress/scripts/tlsalpn.sh +++ b/custom-domain/dstack-ingress/scripts/tlsalpn.sh @@ -269,14 +269,22 @@ APPLY_PENDING=true # still spent one of Let's Encrypt's five failed validations per hostname/hour. FIRST_PASS=true +# Empty when the domain has no certificate yet. +cert_checksum() { sha256sum "$(cert_fullchain_path "$1")" 2>/dev/null || true; } + run_pass() { - local domain status failed=0 changed=0 + local domain status failed=0 changed=0 before while IFS= read -r domain; do [[ -n "$domain" ]] || continue + before=$(cert_checksum "$domain") # `if` context, not a bare command: process_domain returns 2 for # "nothing to do", and under `set -e` a bare non-zero would kill the # whole script on every quiet renewal pass. if ( process_domain "$domain" ); then status=0; else status=$?; fi + # lego can be killed by the timeout after saving a certificate. The + # new certificate is on disk by then and the next pass will find it + # current, so apply it now or never. + [ "$(cert_checksum "$domain")" = "$before" ] || changed=1 case $status in 0) changed=1 ;; 2) ;; From 03b3c0ee1ef4551a6e70706962c4e6e6a852473b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 14 Oct 2026 04:54:32 -0700 Subject: [PATCH 2/2] fix(ingress): start serving when any domain has a certificate, not only all Signed-off-by: Kevin Wang --- custom-domain/dstack-ingress/scripts/dns01.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/custom-domain/dstack-ingress/scripts/dns01.sh b/custom-domain/dstack-ingress/scripts/dns01.sh index 8258aa9..25f09bf 100644 --- a/custom-domain/dstack-ingress/scripts/dns01.sh +++ b/custom-domain/dstack-ingress/scripts/dns01.sh @@ -412,16 +412,18 @@ load_dstack_identity # dns-01 needs no inbound traffic, so keep the original order: certificate # first, serving second. A certificate failure here is fatal only when there is -# nothing to serve. With certificates from an earlier run on disk, exiting -# would just restart into the same failure -- hitting the CA again on every -# restart -- while the certificates go unserved; serve them and let the loop -# retry with backoff instead. -have_all_certs() { +# nothing to serve. Otherwise exiting would just restart into the same failure +# -- hitting the CA again on every restart -- while every domain goes unserved, +# including those whose certificates are fine (a newly added domain failing its +# first issuance would take down the others). Serve what there is and let the +# loop retry with backoff instead. +have_any_cert() { local domain while IFS= read -r domain; do [[ -n "$domain" ]] || continue - [ -f "$(cert_fullchain_path "$domain")" ] && [ -f "$(cert_privkey_path "$domain")" ] || return 1 + [ -f "$(cert_fullchain_path "$domain")" ] && [ -f "$(cert_privkey_path "$domain")" ] && return 0 done <<<"$(get-all-domains.sh)" + return 1 } startup_status=0 @@ -430,8 +432,8 @@ case $startup_status in 0) next_check=${RENEW_INTERVAL:-43200} ;; 3) next_check=60 ;; *) - have_all_certs || exit "$startup_status" - echo "Certificate pass failed; serving the existing certificates and retrying" >&2 + have_any_cert || exit "$startup_status" + echo "Certificate pass failed; serving the certificates there are and retrying" >&2 next_check=60 ;; esac