From 9810e69f1a6a50345150bc8edbfe8256d3181039 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 14 Oct 2026 01:12:24 -0700 Subject: [PATCH 1/5] fix(ingress): follow the configured ACME server and account The certificate volume outlives the configuration, and several places assumed it did not: - certbot: after ACME_STAGING true -> false, certonly found the staging lineage not yet due and kept it, and `renew` (no --server) restored the staging server and account from the lineage. Production -> staging failed in the renewal window. certman.py now compares the lineage's `server` with the configured one and re-issues (--force-renewal --break-my-certs) on a mismatch; derived-key certificates record their server alongside. Every certbot command passes --server explicitly. - ACME_DIRECTORY_URL was dropped under DELEGATION_ZONE, and the account lookups (acme_account_exists, functions.sh) hard-coded Let's Encrypt, so with a custom server every pass re-ran certonly and evidence/CAA found no account. certman.py derives the server and its account path in one place; the shell asks `certman.py account-file`. - lego: account_file() took the first account of any email and register() stopped if one existed, so after ACME_EMAIL changed the CAA accounturi and evidence named the old account. tlsalpn.sh globbed accounts across servers, so the staging account sorts first after going to production. legoman.py now resolves accounts//, the account `lego run` uses, and tlsalpn.sh asks `legoman.py account-file`. lego honours ACME_DIRECTORY_URL too. Signed-off-by: Kevin Wang --- .../dstack-ingress/scripts/certman.py | 124 ++++++++++++------ .../dstack-ingress/scripts/functions.sh | 27 +--- .../dstack-ingress/scripts/legoman.py | 45 ++++--- .../scripts/tests/test_certman.py | 66 ++++++++++ .../dstack-ingress/scripts/tlsalpn.sh | 8 +- 5 files changed, 185 insertions(+), 85 deletions(-) diff --git a/custom-domain/dstack-ingress/scripts/certman.py b/custom-domain/dstack-ingress/scripts/certman.py index d596fa6c..d3bcb8c3 100644 --- a/custom-domain/dstack-ingress/scripts/certman.py +++ b/custom-domain/dstack-ingress/scripts/certman.py @@ -2,6 +2,7 @@ import deterministic_key import argparse +import glob import os import re import shutil @@ -11,6 +12,7 @@ from datetime import datetime, timedelta, timezone from importlib import metadata from typing import List, Optional, Tuple +from urllib.parse import urlparse from cryptography import x509 @@ -45,6 +47,27 @@ def staging_enabled() -> bool: return value == "true" +ACME_PROD = "https://acme-v02.api.letsencrypt.org/directory" +ACME_STAGING = "https://acme-staging-v02.api.letsencrypt.org/directory" + + +def acme_server() -> str: + """The ACME directory to use. ACME_DIRECTORY_URL allows local test servers + (for example Pebble) without changing the production/staging defaults.""" + return (os.environ.get("ACME_DIRECTORY_URL", "").strip() + or (ACME_STAGING if staging_enabled() else ACME_PROD)) + + +def account_file() -> Optional[str]: + """certbot's account for acme_server(), stored under the directory URL's + host and path.""" + parsed = urlparse(acme_server()) + pattern = os.path.join(LETSENCRYPT_DIR, "accounts", parsed.netloc + parsed.path, + "*", "regr.json") + matches = sorted(glob.glob(pattern)) + return matches[0] if matches else None + + # Time a certbot run needs on top of the DNS propagation wait: ACME round # trips, plugin setup, the cleanup hook, and certbot's own bookkeeping. CERTBOT_TIMEOUT_HEADROOM = 180 @@ -167,7 +190,7 @@ def setup_credentials(self) -> bool: return result def _build_certbot_command(self, action: str, domain: str, email: str, - csr_args: Tuple[str, ...] = ()) -> List[str]: + extra_args: Tuple[str, ...] = ()) -> List[str]: """Build certbot command using provider configuration.""" certbot_cmd = self._get_certbot_command() @@ -191,15 +214,16 @@ def _build_certbot_command(self, action: str, domain: str, email: str, base_cmd.extend(["--email", email]) else: base_cmd.append("--register-unsafely-without-email") - base_cmd.extend(csr_args) + base_cmd.extend(extra_args) base_cmd.extend(["-d", domain]) # For `renew`, certbot reuses the authenticator + hooks saved in the # renewal config from the initial `certonly`, so we don't re-specify # them here (and must not fall back to the DNS plugin). if action == "renew": base_cmd.extend(["--cert-name", self._lineage_name(domain)]) - if staging_enabled(): - base_cmd.append("--staging") + # Explicit on `renew` too: certbot would otherwise take the server + # saved in the lineage, whatever the configuration says now. + base_cmd.extend(["--server", acme_server()]) masked = [a if not (i > 0 and base_cmd[i - 1] == "--email") else "" for i, a in enumerate(base_cmd)] print(f"Executing (challenge-delegation): {' '.join(masked)}") @@ -225,7 +249,7 @@ def _build_certbot_command(self, action: str, domain: str, email: str, f"Credentials file does not exist: {credentials_file}") if action == "certonly": - base_cmd.extend(csr_args) + base_cmd.extend(extra_args) base_cmd.extend(["--agree-tos", "--no-eff-email"]) # The ACME contact address is optional (RFC 8555 section 7.3), and # it is published: the account document is served as attestation @@ -244,13 +268,7 @@ def _build_certbot_command(self, action: str, domain: str, email: str, # N domains meant N passes over all N lineages, and the result was # then reported against whichever domain happened to ask. base_cmd.extend(["--cert-name", self._lineage_name(domain)]) - if staging_enabled(): - base_cmd.extend(["--staging"]) - # Allow local ACME test servers (for example Pebble) without changing - # the production/staging defaults. - acme_server = os.environ.get("ACME_DIRECTORY_URL", "").strip() - if acme_server: - base_cmd.extend(["--server", acme_server]) + base_cmd.extend(["--server", acme_server()]) if getattr(self.provider, 'CERTBOT_PROPAGATION_SECONDS'): propagation_seconds = self.provider.CERTBOT_PROPAGATION_SECONDS @@ -264,8 +282,11 @@ def _build_certbot_command(self, action: str, domain: str, email: str, return base_cmd - def obtain_certificate(self, domain: str, email: str) -> bool: - """Obtain a new certificate for the domain.""" + def obtain_certificate(self, domain: str, email: str, replace: bool = False) -> bool: + """Obtain a new certificate for the domain. + + replace: re-issue even though the current certificate is not due. + """ print(f"Obtaining certificate for {domain} using {self.provider_type}") # Ensure plugin is installed @@ -287,7 +308,7 @@ def obtain_certificate(self, domain: str, email: str) -> bool: return False key = None - csr_args: Tuple[str, ...] = () + extra_args: Tuple[str, ...] = () if deterministic_key.enabled(): key = deterministic_key.private_key(domain) # certbot refuses to overwrite an explicit --cert-path, so issue @@ -297,11 +318,15 @@ def obtain_certificate(self, domain: str, email: str) -> bool: staging = tempfile.mkdtemp(dir=root, prefix=".issue-") csr = os.path.join(staging, "request.csr") deterministic_key.write_file(csr, deterministic_key.csr_pem(key, domain)) - csr_args = ("--csr", csr) + tuple( + extra_args = ("--csr", csr) + tuple( arg for name in ("cert", "chain", "fullchain") for arg in (f"--{name}-path", os.path.join(staging, f"{name}.pem"))) + elif replace: + # certbot keeps a lineage that is not yet due, and refuses to + # replace a trusted certificate with a staging one. + extra_args = ("--force-renewal", "--break-my-certs") - cmd = self._build_certbot_command("certonly", domain, email, csr_args) + cmd = self._build_certbot_command("certonly", domain, email, extra_args) timeout = self._certbot_timeout() try: @@ -357,6 +382,7 @@ def _install_deterministic(self, domain: str, staging: str, key) -> bool: deterministic_key.key_pem(key)) for name in ("cert.pem", "chain.pem", "fullchain.pem"): os.replace(os.path.join(staging, name), os.path.join(target, name)) + deterministic_key.write_file(os.path.join(target, "server"), acme_server().encode()) return True def _clear_legacy_live(self, domain: str) -> None: @@ -492,6 +518,22 @@ def _renewal_conf_path(self, domain: str) -> str: """Where certbot keeps this lineage's renewal config.""" return os.path.join(LETSENCRYPT_DIR, "renewal", f"{self._lineage_name(domain)}.conf") + def _issuing_server(self, domain: str) -> Optional[str]: + """The ACME server the current certificate came from.""" + try: + if deterministic_key.enabled(): + with open(os.path.join(self.cert_dir(domain), "server"), + encoding="utf-8") as fh: + return fh.read().strip() + with open(self._renewal_conf_path(domain), encoding="utf-8") as fh: + for line in fh: + match = re.match(r"\s*server\s*=\s*(\S+)", line) + if match: + return match.group(1) + except OSError: + pass + return None + def _certbot_timeout(self) -> int: """How long one certbot run may take. @@ -631,21 +673,8 @@ def apply_renewal_window(self, domain: str) -> None: print(f"Renewal window for {domain} set to {days} days before expiry") def acme_account_exists(self) -> bool: - """Check if an ACME account exists for the current server (staging or production). - - The account directory differs between staging and production: - - production: /etc/letsencrypt/accounts/acme-v02.api.letsencrypt.org/directory/ - - staging: /etc/letsencrypt/accounts/acme-staging-v02.api.letsencrypt.org/directory/ - - When switching between staging and production, the cert file persists - on the volume but the account only exists for the previous server. - """ - import glob - api_endpoint = "acme-v02.api.letsencrypt.org" - if staging_enabled(): - api_endpoint = "acme-staging-v02.api.letsencrypt.org" - pattern = os.path.join(LETSENCRYPT_DIR, "accounts", api_endpoint, "directory", "*", "regr.json") - return len(glob.glob(pattern)) > 0 + """Check if an ACME account exists for the current server.""" + return account_file() is not None def run_action( self, domain: str, email: str, action: str = "auto" @@ -657,15 +686,24 @@ def run_action( """ self._clear_legacy_live(domain) if action == "auto": - if self.certificate_exists(domain) and self.acme_account_exists(): + if not self.certificate_exists(domain): + success = self.obtain_certificate(domain, email) + return success, success # Always generate evidence for new certificates + # The certificate volume outlives the configuration: after a switch + # between staging and production (or ACME_DIRECTORY_URL), the + # certificate in place came from the other server. + server = self._issuing_server(domain) + if server == acme_server() and self.acme_account_exists(): success, renewed = self.renew_certificate(domain, email) return success, renewed # Only generate evidence if actually renewed + if server != acme_server(): + print(f"Certificate for {domain} came from {server or 'an unknown ACME server'}, " + f"replacing it from {acme_server()}") else: - if self.certificate_exists(domain) and not self.acme_account_exists(): - print(f"Certificate exists for {domain} but ACME account is missing " - f"(staging/production switch?), re-obtaining") - success = self.obtain_certificate(domain, email) - return success, success # Always generate evidence for new certificates + print(f"Certificate exists for {domain} but the ACME account is missing, " + f"re-obtaining") + success = self.obtain_certificate(domain, email, replace=True) + return success, success elif action == "obtain": success = self.obtain_certificate(domain, email) return success, success @@ -681,7 +719,8 @@ def main(): description="Manage SSL certificates with certbot using DNS providers" ) parser.add_argument( - "action", choices=["obtain", "renew", "auto", "setup", "cert-dir"], + "action", choices=["obtain", "renew", "auto", "setup", "cert-dir", + "account-file"], help="Action to perform" ) parser.add_argument("--domain", help="Domain name") @@ -691,6 +730,13 @@ def main(): args = parser.parse_args() + if args.action == "account-file": + path = account_file() + if not path: + print(f"Error: no ACME account found for {acme_server()}", file=sys.stderr) + sys.exit(1) + print(path) + return if args.action == "cert-dir": if not args.domain: parser.error("--domain is required") diff --git a/custom-domain/dstack-ingress/scripts/functions.sh b/custom-domain/dstack-ingress/scripts/functions.sh index df8b5ebc..0d502abb 100644 --- a/custom-domain/dstack-ingress/scripts/functions.sh +++ b/custom-domain/dstack-ingress/scripts/functions.sh @@ -166,31 +166,10 @@ sanitize_proxy_buffers() { fi } -get_letsencrypt_account_path() { - local base_path="/etc/letsencrypt/accounts" - local api_endpoint="acme-v02.api.letsencrypt.org" - - # entrypoint.sh normalises CERTBOT_STAGING into ACME_STAGING. - if [[ "$ACME_STAGING" == "true" ]]; then - api_endpoint="acme-staging-v02.api.letsencrypt.org" - fi - - echo "${base_path}/${api_endpoint}/directory/*/regr.json" -} - +# certman.py decides which ACME server is in use (ACME_STAGING, +# ACME_DIRECTORY_URL), and so where its account lives. get_letsencrypt_account_file() { - local account_pattern - account_pattern=$(get_letsencrypt_account_path) - - local account_files - account_files=( $account_pattern ) - - if [[ ! -f "${account_files[0]}" ]]; then - echo "Error: Let's Encrypt account file not found at $account_pattern" >&2 - return 1 - fi - - echo "${account_files[0]}" + certman.py account-file } txt_record_name() { diff --git a/custom-domain/dstack-ingress/scripts/legoman.py b/custom-domain/dstack-ingress/scripts/legoman.py index 978fb700..87c103ee 100644 --- a/custom-domain/dstack-ingress/scripts/legoman.py +++ b/custom-domain/dstack-ingress/scripts/legoman.py @@ -25,7 +25,6 @@ """ import argparse -import glob import json import os import subprocess @@ -39,6 +38,8 @@ LEGO_PATH = os.environ.get("LEGO_PATH", "/etc/letsencrypt/lego") ACME_PROD = "https://acme-v02.api.letsencrypt.org/directory" ACME_STAGING = "https://acme-staging-v02.api.letsencrypt.org/directory" +# lego's account directory name when there is no email. +NO_EMAIL_ACCOUNT = "noemail@example.com" # Exit codes: 0 changed, 2 nothing to do, 1 failed. The caller reloads on 0. EXIT_CHANGED = 0 @@ -50,6 +51,10 @@ def acme_server() -> str: + # Same override as certman.py, for local test servers such as Pebble. + custom = os.environ.get("ACME_DIRECTORY_URL", "").strip() + if custom: + return custom # CERTBOT_STAGING is the historical name and still works, but there is no # certbot on this path: the ACME client here is lego. staging = os.environ.get("ACME_STAGING") or os.environ.get("CERTBOT_STAGING", "false") @@ -63,21 +68,20 @@ def server_dir() -> str: return f"{host}_{parsed.port}" if parsed.port else host -def account_file() -> Optional[str]: - """Find the account document by globbing rather than by building the path. +def account_file(email: str) -> str: + """The account `lego run` uses: lego keys accounts by server and email, with + a placeholder of its own when there is no email. - lego names the directory after the account email, and after a placeholder of - its own choosing when there is no email. Globbing means we neither have to - know that placeholder nor track it across lego versions. + Any other account in the directory -- one for a previous ACME_EMAIL, say -- + is not the one issuing, so publishing it would name the wrong account. """ - pattern = os.path.join(LEGO_PATH, "accounts", server_dir(), "*", "account.json") - matches = sorted(glob.glob(pattern)) - return matches[0] if matches else None + return os.path.join(LEGO_PATH, "accounts", server_dir(), + email or NO_EMAIL_ACCOUNT, "account.json") -def account_uri() -> Optional[str]: - path = account_file() - if not path: +def account_uri(email: str) -> Optional[str]: + path = account_file(email) + if not os.path.isfile(path): return None try: with open(path, encoding="utf-8") as fh: @@ -135,7 +139,7 @@ def _run(cmd: List[str], timeout: int = RUN_TIMEOUT) -> Tuple[int, str]: def register(email: str) -> int: """Create the ACME account without issuing anything.""" - if account_file(): + if os.path.isfile(account_file(email)): print("ACME account already exists") return EXIT_UNCHANGED @@ -144,7 +148,7 @@ def register(email: str) -> int: [LEGO_BIN, "accounts", "register"] + _common_flags(email), timeout=REGISTER_TIMEOUT, ) - if code == 0 and account_file(): + if code == 0 and os.path.isfile(account_file(email)): print("✓ ACME account registered") return EXIT_CHANGED print(f"✗ ACME account registration failed (exit code {code})", file=sys.stderr) @@ -242,7 +246,8 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "action", - choices=["obtain", "renew", "auto", "register", "account-uri", "cert-path"], + choices=["obtain", "renew", "auto", "register", "account-uri", "account-file", + "cert-path"], ) parser.add_argument("--domain", help="Domain name") parser.add_argument("--email", help="Email for ACME registration") @@ -251,12 +256,20 @@ def main() -> int: email = args.email or os.environ.get("ACME_EMAIL") or os.environ.get("CERTBOT_EMAIL", "") if args.action == "account-uri": - uri = account_uri() + uri = account_uri(email) if not uri: return EXIT_ERROR print(uri) return EXIT_CHANGED + if args.action == "account-file": + path = account_file(email) + if not os.path.isfile(path): + print(f"Error: lego account file not found at {path}", file=sys.stderr) + return EXIT_ERROR + print(path) + return EXIT_CHANGED + if args.action == "cert-path": if not args.domain: print("Error: --domain is required", file=sys.stderr) diff --git a/custom-domain/dstack-ingress/scripts/tests/test_certman.py b/custom-domain/dstack-ingress/scripts/tests/test_certman.py index 367ebda1..35709972 100644 --- a/custom-domain/dstack-ingress/scripts/tests/test_certman.py +++ b/custom-domain/dstack-ingress/scripts/tests/test_certman.py @@ -183,6 +183,8 @@ class EnvTestCase(unittest.TestCase): "DELEGATION_PROPAGATION_SECONDS", "ACME_STAGING", "CERTBOT_STAGING", + "ACME_DIRECTORY_URL", + "DETERMINISTIC_TLS_KEY", ) def setUp(self): @@ -229,6 +231,70 @@ def test_delegation_renew_is_scoped_too(self): self.assertNotIn("--manual", cmd) +class AcmeServerTest(EnvTestCase): + """The configured ACME server, not the one saved in a lineage, decides. + + The certificate volume outlives the configuration, so after a switch + between staging and production the certificate in place, its renewal + config and its account all belong to the other server. + """ + + COMMANDS = (("certonly", False), ("renew", False), ("certonly", True), ("renew", True)) + + def servers(self): + out = [] + for action, delegated in self.COMMANDS: + if delegated: + os.environ["DELEGATION_ZONE"] = "deleg.example.net" + else: + os.environ.pop("DELEGATION_ZONE", None) + cmd = _command_manager()._build_certbot_command(action, "a.example.com", "") + self.assertNotIn("--staging", cmd) + out.append(cmd[cmd.index("--server") + 1]) + return set(out) + + def test_every_command_names_the_server(self): + self.assertEqual(self.servers(), {certman.ACME_PROD}) + os.environ["ACME_STAGING"] = "true" + self.assertEqual(self.servers(), {certman.ACME_STAGING}) + + def test_directory_url_applies_to_every_command(self): + os.environ["ACME_STAGING"] = "true" + os.environ["ACME_DIRECTORY_URL"] = "https://pebble:14000/dir" + self.assertEqual(self.servers(), {"https://pebble:14000/dir"}) + + def run_auto(self, lineage_server, account=True): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + conf = os.path.join(tmp, "a.example.com.conf") + with open(conf, "w", encoding="utf-8") as fh: + fh.write(f"version = 5.8.0\n[renewalparams]\nserver = {lineage_server}\n") + calls = [] + mgr = object.__new__(certman.CertManager) + mgr._renewal_conf_path = lambda domain: conf # noqa: SLF001 + mgr.certificate_exists = lambda domain: True + mgr.acme_account_exists = lambda: account + mgr.renew_certificate = lambda d, e: calls.append("renew") or (True, False) + mgr.obtain_certificate = lambda d, e, replace=False: calls.append( + ("obtain", replace)) or True + mgr.run_action("a.example.com", "", "auto") + return calls + + def test_same_server_renews(self): + self.assertEqual(self.run_auto(certman.ACME_PROD), ["renew"]) + + def test_staging_to_production_replaces_the_certificate(self): + self.assertEqual(self.run_auto(certman.ACME_STAGING), [("obtain", True)]) + + def test_production_to_staging_replaces_the_certificate(self): + os.environ["ACME_STAGING"] = "true" + self.assertEqual(self.run_auto(certman.ACME_PROD), [("obtain", True)]) + + def test_missing_account_replaces_the_certificate(self): + self.assertEqual(self.run_auto(certman.ACME_PROD, account=False), + [("obtain", True)]) + + class CertbotTimeoutTest(EnvTestCase): """A fixed 300s cap could not fit every provider's own propagation wait.""" diff --git a/custom-domain/dstack-ingress/scripts/tlsalpn.sh b/custom-domain/dstack-ingress/scripts/tlsalpn.sh index b42f7958..80b35483 100644 --- a/custom-domain/dstack-ingress/scripts/tlsalpn.sh +++ b/custom-domain/dstack-ingress/scripts/tlsalpn.sh @@ -42,13 +42,9 @@ esac cert_fullchain_path() { echo "${LEGO_PATH}/certificates/${1}.crt"; } cert_privkey_path() { echo "${LEGO_PATH}/certificates/${1}.key"; } +# The account for the configured server and email, as legoman.py resolves it. acme_account_file() { - local files=( "${LEGO_PATH}"/accounts/*/*/account.json ) - if [[ ! -f "${files[0]}" ]]; then - echo "Error: lego account file not found under ${LEGO_PATH}/accounts" >&2 - return 1 - fi - echo "${files[0]}" + legoman.py account-file ${ACME_EMAIL:+--email "$ACME_EMAIL"} } setup_py_env() { From 40b2b4153d289c25fb9fd7c1c659ceb58f454447 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 14 Oct 2026 02:49:37 -0700 Subject: [PATCH 2/5] fix(ingress): move lego off the previous CA, and converge on unknown servers lego compared nothing but the remaining lifetime, so after ACME_STAGING or ACME_DIRECTORY_URL changed it kept serving the old CA's certificate, and once that was due every renewal failed: lego's ARI request names a certificate the new CA never issued. legoman.py now compares the CA in the certUrl lego records with the configured one and re-issues with --renew-force --ari-disable on a mismatch. certman.py re-issued on every pass when a lineage's renewal config named no server, which re-issuing does not fix. certbot always records it, so only a derived-key certificate from before the server was recorded is treated as unknown; it is re-issued once, which records the server. Signed-off-by: Kevin Wang --- .../dstack-ingress/scripts/certman.py | 4 +++ .../dstack-ingress/scripts/legoman.py | 28 ++++++++++++++++++- .../scripts/tests/test_certman.py | 6 +++- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/custom-domain/dstack-ingress/scripts/certman.py b/custom-domain/dstack-ingress/scripts/certman.py index d3bcb8c3..c5c2f006 100644 --- a/custom-domain/dstack-ingress/scripts/certman.py +++ b/custom-domain/dstack-ingress/scripts/certman.py @@ -693,6 +693,10 @@ def run_action( # between staging and production (or ACME_DIRECTORY_URL), the # certificate in place came from the other server. server = self._issuing_server(domain) + if server is None and not deterministic_key.enabled(): + # certbot always records the server; a lineage without one has + # a broken renewal config, which re-issuing would not repair. + server = acme_server() if server == acme_server() and self.acme_account_exists(): success, renewed = self.renew_certificate(domain, email) return success, renewed # Only generate evidence if actually renewed diff --git a/custom-domain/dstack-ingress/scripts/legoman.py b/custom-domain/dstack-ingress/scripts/legoman.py index 87c103ee..605a07a3 100644 --- a/custom-domain/dstack-ingress/scripts/legoman.py +++ b/custom-domain/dstack-ingress/scripts/legoman.py @@ -105,6 +105,22 @@ def certificate_exists(domain: str) -> bool: return os.path.isfile(crt) and os.path.isfile(key) +def _origin(url: str) -> str: + parsed = urlparse(url) + return f"{parsed.scheme}://{parsed.netloc}" + + +def issuing_origin(domain: str) -> Optional[str]: + """The CA the current certificate came from, by the URL lego recorded.""" + path = os.path.join(LEGO_PATH, "certificates", f"{domain}.json") + try: + with open(path, encoding="utf-8") as fh: + url = json.load(fh).get("certUrl") + except (OSError, ValueError, AttributeError): + return None + return _origin(url) if url else None + + def _renewed_marker(domain: str) -> str: return os.path.join(LEGO_PATH, f".renewed-{domain}") @@ -188,6 +204,7 @@ def run_cert(domain: str, email: str) -> int: crt, key_file = cert_paths(domain) key = None target = ["--domains", domain] + force: List[str] = [] if deterministic_key.enabled(): # With --csr lego renews on its usual schedule but never rewrites the # .key file, so it is kept in step below. @@ -198,12 +215,21 @@ def run_cert(domain: str, email: str) -> int: target = ["--csr", f"{base}.csr", "--private-key", f"{base}.key"] if certificate_exists(domain) and not deterministic_key.cert_matches(crt, key): print(f"Certificate for {domain} does not carry the derived key, re-issuing") - target.append("--renew-force") + force = ["--renew-force"] + + # The certificate volume outlives the configuration. After a switch of CA, + # lego would keep the old CA's certificate until it is due, and then fail: + # its ARI request names a certificate the new CA never issued. + origin = issuing_origin(domain) + if certificate_exists(domain) and origin and origin != _origin(acme_server()): + print(f"Certificate for {domain} came from {origin}, replacing it from {acme_server()}") + force = ["--renew-force", "--ari-disable"] cmd = ( [LEGO_BIN, "run"] + _common_flags(email) + target + + force + _challenge_flags() + ["--deploy-hook", f"touch {marker}"] ) diff --git a/custom-domain/dstack-ingress/scripts/tests/test_certman.py b/custom-domain/dstack-ingress/scripts/tests/test_certman.py index 35709972..87700161 100644 --- a/custom-domain/dstack-ingress/scripts/tests/test_certman.py +++ b/custom-domain/dstack-ingress/scripts/tests/test_certman.py @@ -268,7 +268,8 @@ def run_auto(self, lineage_server, account=True): self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) conf = os.path.join(tmp, "a.example.com.conf") with open(conf, "w", encoding="utf-8") as fh: - fh.write(f"version = 5.8.0\n[renewalparams]\nserver = {lineage_server}\n") + if lineage_server: + fh.write(f"version = 5.8.0\n[renewalparams]\nserver = {lineage_server}\n") calls = [] mgr = object.__new__(certman.CertManager) mgr._renewal_conf_path = lambda domain: conf # noqa: SLF001 @@ -294,6 +295,9 @@ def test_missing_account_replaces_the_certificate(self): self.assertEqual(self.run_auto(certman.ACME_PROD, account=False), [("obtain", True)]) + def test_unknown_server_does_not_replace_every_pass(self): + self.assertEqual(self.run_auto(None), ["renew"]) + class CertbotTimeoutTest(EnvTestCase): """A fixed 300s cap could not fit every provider's own propagation wait.""" From 0790ffd7e45f6843c167e278be8120d846b5cd8c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 14 Oct 2026 04:19:48 -0700 Subject: [PATCH 3/5] docs(ingress): document ACME_DIRECTORY_URL Signed-off-by: Kevin Wang --- custom-domain/dstack-ingress/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index 1c9c81e9..1004ad9d 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -178,6 +178,7 @@ otherwise arbitrary — the gateway answers every name under its domain. | `SET_CAA` | `false` | Enable CAA DNS record (dns-01 only; tls-alpn-01 cannot write DNS) | | `TXT_PREFIX` | `_dstack-app-address` | DNS TXT record prefix | | `ACME_STAGING` | `false` | Use Let's Encrypt staging, in either mode. `CERTBOT_STAGING` is the historical name and still works | +| `ACME_DIRECTORY_URL` | | ACME directory to use instead of Let's Encrypt (e.g. Pebble), in either mode; takes precedence over `ACME_STAGING` | | `CHALLENGE_TYPE` | `dns-01` | `dns-01` (certbot + DNS credentials) or `tls-alpn-01` (lego, no DNS credentials) | | `DNS_SETUP_MODE` | `wait` | How to handle operator-managed records: `wait`, `print` or `webhook` — see below. Applies to tls-alpn-01 and to dns-01 challenge delegation | | `DNS_SETUP_TIMEOUT` | `1800` | Seconds to wait for those records to appear | From 3f71e6c00b2c3e681eb5fb2857c2c92745ffd106 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 14 Oct 2026 04:50:53 -0700 Subject: [PATCH 4/5] docs(ingress): scope ACME_DIRECTORY_URL to test servers Signed-off-by: Kevin Wang --- custom-domain/dstack-ingress/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index 1004ad9d..d8cf83c7 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -178,7 +178,7 @@ otherwise arbitrary — the gateway answers every name under its domain. | `SET_CAA` | `false` | Enable CAA DNS record (dns-01 only; tls-alpn-01 cannot write DNS) | | `TXT_PREFIX` | `_dstack-app-address` | DNS TXT record prefix | | `ACME_STAGING` | `false` | Use Let's Encrypt staging, in either mode. `CERTBOT_STAGING` is the historical name and still works | -| `ACME_DIRECTORY_URL` | | ACME directory to use instead of Let's Encrypt (e.g. Pebble), in either mode; takes precedence over `ACME_STAGING` | +| `ACME_DIRECTORY_URL` | | ACME directory of a test server such as Pebble, used instead of Let's Encrypt in either mode (CAA records always name `letsencrypt.org`, so other public CAs are not supported); takes precedence over `ACME_STAGING` | | `CHALLENGE_TYPE` | `dns-01` | `dns-01` (certbot + DNS credentials) or `tls-alpn-01` (lego, no DNS credentials) | | `DNS_SETUP_MODE` | `wait` | How to handle operator-managed records: `wait`, `print` or `webhook` — see below. Applies to tls-alpn-01 and to dns-01 challenge delegation | | `DNS_SETUP_TIMEOUT` | `1800` | Seconds to wait for those records to appear | From a05bf60c3e0264faa26966b941511ab2634caa2a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 15 Oct 2026 09:34:28 +0800 Subject: [PATCH 5/5] Potential fix for pull request finding 'CodeQL / Empty except' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- custom-domain/dstack-ingress/scripts/certman.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/custom-domain/dstack-ingress/scripts/certman.py b/custom-domain/dstack-ingress/scripts/certman.py index c5c2f006..e7f42e99 100644 --- a/custom-domain/dstack-ingress/scripts/certman.py +++ b/custom-domain/dstack-ingress/scripts/certman.py @@ -530,8 +530,8 @@ def _issuing_server(self, domain: str) -> Optional[str]: match = re.match(r"\s*server\s*=\s*(\S+)", line) if match: return match.group(1) - except OSError: - pass + except OSError as err: + print(f"Warning: unable to read issuing server for {domain}: {err}", file=sys.stderr) return None def _certbot_timeout(self) -> int: