diff --git a/.env.bootstrap.example b/.env.bootstrap.example index 2f75a7f..2ddc6a3 100644 --- a/.env.bootstrap.example +++ b/.env.bootstrap.example @@ -24,8 +24,9 @@ K3S_JOIN_SERVER_URL= K3S_AGENT_SERVER_URL= K3S_AGENT_NODE_NAME= -# k3s image version (rancher/k3s tag). Empty means latest. +# The rancher/k3s image tag (for example v1.34.1-k3s1) and the tailscale/tailscale image tag (for example v1.102.4) the node is built from. Empty means the role's pinned defaults; never latest. K3S_VERSION= +TAILSCALE_VERSION= # The directory the role keeps this node's Compose project, .env and kubeconfig in. Empty means ~/.github-runner. GITHUB_RUNNER_CLUSTER_DIR= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2841d1e..a611e5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,7 +83,7 @@ jobs: mapfile -t dockerfiles < <(find . -type f -iname 'Dockerfile*' -not -path './.git/*') echo "Found ${#dockerfiles[@]} Dockerfile(s)." if [ "${#dockerfiles[@]}" -gt 0 ]; then - # error threshold, not the default (info): this repo deliberately floats several base images/packages on `latest` rather than pinning (see e.g. ghcr.io/exadev/github-runner:latest, rancher/k3s:${K3S_VERSION:-latest}), so hadolint's warning-level pinning advice (DL3007/DL3008/DL3018) stays visible in the log without failing every run over an already-made, documented tradeoff. + # error threshold, not the default (info): this repo deliberately floats several base images/packages on `latest` rather than pinning (see e.g. ghcr.io/exadev/github-runner:latest), so hadolint's warning-level pinning advice (DL3007/DL3008/DL3018) stays visible in the log without failing every run over an already-made, documented tradeoff. ./hadolint --failure-threshold error "${dockerfiles[@]}" fi @@ -112,8 +112,13 @@ jobs: tar -tzf "${RUNNER_TEMP}"/collection/*.tar.gz | grep -Fx roles/github_runner_cluster/files/docker-compose.yml tar -tzf "${RUNNER_TEMP}"/collection/*.tar.gz | grep -Fx roles/github_runner_cluster/files/k3s/node-entrypoint.sh - name: Unit-test the cluster role's server selection - # Ubuntu's ansible package, installed above, puts ansible-core on the system Python the filter plugin imports from. - run: python3 -m unittest discover -s tests/unit -v + # Ubuntu's ansible package, installed above, puts ansible-core on the system Python the filter plugins import from. The runner image can carry another Ansible earlier on PATH, so the tests run with /usr/bin first and ANSIBLE_PLAYBOOK pointing at Ubuntu's, which makes every playbook a test runs use the same ansible-core as the filters it imports; both versions are printed for the record. + run: | + set -euo pipefail + export PATH="/usr/bin:${PATH}" ANSIBLE_PLAYBOOK=/usr/bin/ansible-playbook + ansible-playbook --version | head -n 1 + python3 -c 'import ansible.release; print("ansible-core", ansible.release.__version__)' + python3 -m unittest discover -s tests/unit -v ansible-lint: name: ansible-lint diff --git a/bootstrap.sh b/bootstrap.sh index 7b18605..640da73 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -55,7 +55,6 @@ jq -n \ github_runner_cluster_node_name: $node_name, github_runner_cluster_server_url: (if $node_role == "agent" then env.K3S_AGENT_SERVER_URL else (env.K3S_JOIN_SERVER_URL // "") end), github_runner_cluster_tls_sans: ((env.K3S_TLS_SAN_LIST // "") | split(" ") | map(select(. != ""))), - github_runner_cluster_k3s_version: (env.K3S_VERSION // "" | if . == "" then "latest" else . end), github_runner_arc_heartbeat_gist_id: (env.HEARTBEAT_GIST_ID // ""), github_runner_arc_orgs: ( if (env.RUNNER_APP_ID // "") == "" then [] @@ -81,6 +80,8 @@ jq -n \ github_runner_arc_heartbeat_gh_token: (env.HEARTBEAT_GH_TOKEN // "") } + opt("github_runner_cluster_dir"; "GITHUB_RUNNER_CLUSTER_DIR") + + opt("github_runner_cluster_k3s_version"; "K3S_VERSION") + + opt("github_runner_cluster_tailscale_version"; "TAILSCALE_VERSION") + opt("github_runner_cluster_compose_project"; "GITHUB_RUNNER_COMPOSE_PROJECT") + opt("github_runner_arc_autoscaler_usable_budget_gi"; "AUTOSCALER_USABLE_BUDGET_GI") + opt("github_runner_arc_autoscaler_max_ceiling"; "AUTOSCALER_MAX_CEILING") diff --git a/plugins/filter/cluster_versions.py b/plugins/filter/cluster_versions.py new file mode 100644 index 0000000..77152e3 --- /dev/null +++ b/plugins/filter/cluster_versions.py @@ -0,0 +1,171 @@ +"""Filters the github_runner_cluster role uses to check the versions every node's k3s image is built from: that each is a pinned tag, that every host in a cluster group names the same k3s minor version, and that the version keeps to k3s's upgrade path and Kubernetes' version skew policy from what the cluster runs.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence +from typing import Any + +from ansible.errors import AnsibleFilterError + +# A rancher/k3s image tag: k3s's own version with "+" written as "-", for example v1.34.1-k3s1. +K3S_TAG = re.compile(r"^v(?P[0-9]+)\.(?P[0-9]+)\.(?P[0-9]+)-k3s(?P[0-9]+)$") +# A tailscale/tailscale image tag for a release, for example v1.102.4. +TAILSCALE_TAG = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+$") + + +def cluster_version_errors(k3s_version: Any, tailscale_version: Any) -> list[str]: + """Say what is wrong with the image versions a node is to be built from, or return an empty list. + + :param k3s_version: the rancher/k3s tag, for example v1.34.1-k3s1. + :param tailscale_version: the tailscale/tailscale tag, for example v1.102.4. + :returns: one sentence per problem. + """ + errors: list[str] = [] + if not isinstance(k3s_version, str) or K3S_TAG.match(k3s_version.strip()) is None: + errors.append( + f"github_runner_cluster_k3s_version is {k3s_version!r}; it must be a pinned rancher/k3s tag such as v1.34.1-k3s1" + " (k3s's version with '+' written as '-'), never latest, so that every node is built from the same k3s." + ) + if not isinstance(tailscale_version, str) or TAILSCALE_TAG.match(tailscale_version.strip()) is None: + errors.append(f"github_runner_cluster_tailscale_version is {tailscale_version!r}; it must be a pinned tailscale/tailscale release tag such as v1.102.4, never latest.") + return errors + + +def cluster_k3s_minor_skew(hosts: Any, hostvars: Any, default: Any) -> list[str]: + """Return every host's k3s version when the hosts of a cluster group do not all name the same k3s minor version, or an empty list when they do. + + :param hosts: the cluster group's host names. + :param hostvars: Ansible's hostvars, which hold each host's inventory and extra variables but never role defaults; a host without github_runner_cluster_k3s_version of its own uses ``default``. + :param default: the version that applies where a host sets none: the role's own default (github_runner_cluster_k3s_tested_version), which is the same whichever host runs the check. Never the running host's github_runner_cluster_k3s_version, which would make a host that sets another minor see that minor everywhere and pass. + :returns: ``["host: version", ...]`` for every host when more than one minor version is named, otherwise ``[]``. A version that is not a k3s tag counts as a minor of its own. + """ + if not isinstance(hosts, Sequence) or isinstance(hosts, str) or not isinstance(hostvars, Mapping): + raise AnsibleFilterError("The hosts must be a list and hostvars a mapping.") + versions: dict[str, str] = {} + for host in hosts: + own = hostvars.get(host, {}).get("github_runner_cluster_k3s_version") if isinstance(hostvars.get(host, {}), Mapping) else None + versions[str(host)] = str(own if own is not None else default).strip() + minors = set() + for version in versions.values(): + match = K3S_TAG.match(version) + minors.add((match.group("major"), match.group("minor")) if match else version) + if len(minors) <= 1: + return [] + return [f"{host}: {version}" for host, version in versions.items()] + + +# A version k3s reports: `k3s --version` prints "k3s version v1.34.1+k3s1 (24fc436e)" and a Node's kubeletVersion is "v1.34.1+k3s1". +RUNNING_VERSION = re.compile(r"\bv(?P[0-9]+)\.(?P[0-9]+)\.(?P[0-9]+)\+k3s(?P[0-9]+)\b") + + +def _release(match: re.Match[str]) -> tuple[int, int, int, int]: + """A k3s release as comparable numbers: major, minor, patch and k3s's own build.""" + return (int(match.group("major")), int(match.group("minor")), int(match.group("patch")), int(match.group("build"))) + + +# How ansible-core reports a task cut off by its timeout keyword: from 2.18 the result has a "timedout" key; before that only its message says so. +TIMEOUT_MESSAGES = ("failed to execute in the expected time frame", "Timed out after") + + +def _server_read(result: Any) -> tuple[str, str, list[str]]: + """Classify one server's read of the cluster (an Ansible loop result of tasks/running_version.yml's read). + + :returns: the host, its state ("answered", "stopped" or "unread") and, for "answered", the node lines; for "unread", one line saying why. + """ + if not isinstance(result, Mapping): + raise AnsibleFilterError(f"{result!r} is not a loop result.") + host = str(result.get("item", "?")) + message = str(result.get("msg", "") or "") + if result.get("unreachable"): + return host, "unread", [f"{host} could not be reached ({message.strip().splitlines()[0] if message.strip() else 'no reason given'})"] + if result.get("timedout") or any(text in message for text in TIMEOUT_MESSAGES): + return host, "unread", [f"{host} did not answer within github_runner_cluster_cluster_read_timeout"] + lines = [line.strip() for line in str(result.get("stdout", "")).splitlines() if line.strip()] + if result.get("rc") != 0: + reason = [line.strip() for line in str(result.get("stderr", "") or message).splitlines() if line.strip()] + return host, "unread", [f"{host} could not be read ({reason[-1] if reason else 'exit status ' + str(result.get('rc'))})"] + if lines[:1] == ["stopped"]: + return host, "stopped", [] + if lines[:1] == ["running"]: + return host, "answered", lines[1:] + return host, "unread", [f"{host} gave an answer this check cannot read ({' '.join(lines)!r})"] + + +def cluster_k3s_version_change_blockers(k3s_version: Any, node_name: Any, own_report: Any, server_reads: Any, node_role: Any = "server", builds: Any = True) -> list[str]: + """Say why building ``k3s_version`` would move a node outside k3s's upgrade path and Kubernetes' version skew policy, or return an empty list. + + The rules, from k3s (no downgrade, the cluster upgraded one minor at a time, servers before agents) and Kubernetes' version skew policy (API servers within one minor of each other, a kubelet never newer than any API server, older ones supported): + + - this node is never built older than it runs, release by release (patch and k3s build too); + - no node is built more than one minor above any node of the cluster; + - a server is never built an older minor than any other node runs, server or agent; + - an agent is never built a newer minor than any server runs. This one applies only when the run builds the agent's image (``builds``): changing the pin builds nothing, so writing a new pin to an agent before its servers are rebuilt, as the documented upgrade does, is allowed. + + Patch releases on other nodes are not compared: a node may be a patch behind or ahead of its peers. + + :param k3s_version: the rancher/k3s tag this host's image is built from, for example v1.34.1-k3s1; it must already have passed ``cluster_version_errors``. + :param node_name: this host's node name: its container, and its own entry in the node list, are this node. + :param own_report: what ``k3s --version`` printed in this node's running container (for example "k3s version v1.34.1+k3s1 (24fc436e)"), or an empty string when it does not run. + :param server_reads: the loop results of reading the cluster through each server of the inventory (tasks/running_version.yml), each with its host as ``item`` and ``stdout`` either "stopped" or "running" followed by " " lines (the label "true" on a server, empty on an agent); an empty list when the inventory holds no server. + :param node_role: this host's role, "server" or "agent". + :param builds: whether this run builds the node's image (it is missing), which is when an agent's kubelet would take the pinned version. + :returns: one sentence per rule broken. A version that cannot be read is a problem too, since the check would otherwise pass on what it never saw, and so is a cluster that some server runs but no server could be read through. + """ + pinned = K3S_TAG.match(str(k3s_version).strip()) + if pinned is None: + raise AnsibleFilterError(f"{k3s_version!r} is not a rancher/k3s tag.") + if not isinstance(own_report, str) or not isinstance(server_reads, Sequence) or isinstance(server_reads, str): + raise AnsibleFilterError("The own report must be a string and the server reads a list.") + if node_role not in ("server", "agent"): + raise AnsibleFilterError(f"{node_role!r} is not a node role (server or agent).") + reads = [_server_read(result) for result in server_reads] + answered = [lines for _, state, lines in reads if state == "answered"] + unread = [line for _, state, lines in reads if state == "unread" for line in lines] + problems: list[str] = [] + if unread and not answered: + problems.append(f"The cluster could not be read through any server of the inventory, so what it runs cannot be told: {'; '.join(unread)}.") + # (source, version, is this node, is a server) + running: list[tuple[str, str, bool, bool]] = [] + if own_report.strip(): + running.append((f"This node's own container ({node_name})", own_report.strip(), True, node_role == "server")) + seen: set[str] = set() + for lines in answered: + for line in lines: + fields = line.split() + if not fields or fields[0] in seen: + continue + seen.add(fields[0]) + running.append((f"Node {fields[0]}", fields[1] if len(fields) > 1 else "", fields[0] == str(node_name), len(fields) > 2 and fields[2] == "true")) + target = _release(pinned) + for source, version, is_self, is_server in running: + found = RUNNING_VERSION.search(version) + if found is None: + problems.append(f"{source} reports {version!r}, which is not a k3s version this check can read.") + continue + current = _release(found) + shown = found.group(0) + if is_self and target < current: + problems.append(f"{source} runs {shown}, newer than {k3s_version}; k3s cannot be downgraded.") + elif target[0] != current[0] or target[1] > current[1] + 1: + problems.append(f"{source} runs {shown}, more than one minor away from {k3s_version}; k3s upgrades the cluster one minor at a time.") + elif node_role == "server" and not is_self and target[:2] < current[:2]: + if is_server: + problems.append(f"Server {source.removeprefix('Node ')} runs {shown}, a newer minor than {k3s_version}; the servers cannot go back a minor.") + else: + problems.append(f"Agent {source.removeprefix('Node ')} runs {shown}, a newer minor than {k3s_version}; a kubelet may not be newer than the API servers.") + elif node_role == "agent" and is_server and bool(builds) and target[:2] > current[:2]: + problems.append(f"Server {source.removeprefix('Node ')} runs {shown}, an older minor than {k3s_version}; an agent's kubelet may not be newer than the API servers, so upgrade the servers first.") + return problems + + +class FilterModule: + """Registers the cluster version filters with Ansible.""" + + def filters(self) -> dict[str, Any]: + """Return the filters this plugin provides.""" + return { + "cluster_version_errors": cluster_version_errors, + "cluster_k3s_minor_skew": cluster_k3s_minor_skew, + "cluster_k3s_version_change_blockers": cluster_k3s_version_change_blockers, + } diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index fa91661..f5ac7f4 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -8,6 +8,44 @@ Which hosts are servers is decided automatically from the inventory group named The role reads `docker compose version` first and fails, before changing anything on the host, if it is 2.37.1 or later but older than 2.39.0. Those releases create a container from an image they have just built without recording that image's ID on it ([docker/compose#13047](https://github.com/docker/compose/pull/13047)), so the next run sees a different image and recreates the container, which restarts every node on the run after the one that built the k3s image. Releases before 2.37.1 and from 2.39.0 on keep the containers. GitHub's ubuntu-latest runner image has shipped 2.38.2, so a CI job that runs the role there needs a different Compose installed first, as `.github/workflows/mesh-integration.yml` does. +## Versions + +Every node's image is built from two pinned tags: `github_runner_cluster_k3s_version` (a `rancher/k3s` release tag, k3s's version with `+` written as `-`, such as `v1.34.1-k3s1`; release candidates are not accepted) and `github_runner_cluster_tailscale_version` (a `tailscale/tailscale` tag such as `v1.102.4`). The defaults are the versions this collection's integration tests run. The role writes both into the host's `.env`, the Compose file refuses to build without them, and the Dockerfile has no default for either, so no node is ever built from `latest`. Set them once for the whole group, in the inventory or `group_vars`, not per host and not in play vars. + +Before it changes anything on a host the role fails: + +- if either is not a pinned tag; +- if the hosts of `github_runner_cluster_group` do not all name the same k3s minor version, listing each host's version. Every host is read from the inventory, and one that sets nothing counts as the default, so the check gives the same answer on every host, including under `--limit`; +- if the k3s version would break k3s's upgrade path or Kubernetes' version skew policy from what runs now. k3s cannot be downgraded and upgrades the cluster one minor at a time, servers before agents; the [version skew policy](https://kubernetes.io/releases/version-skew-policy/) ([archived](http://web.archive.org/web/2026/https://kubernetes.io/releases/version-skew-policy/)) keeps the API servers within one minor of each other and never lets a kubelet be newer than an API server, while older kubelets are supported. So the role refuses to build this node older than it runs (by patch release and k3s build too), any node more than one minor above another, a server an older minor than any node runs, and an agent a newer minor than any server runs. Patch releases on other nodes are not compared. The agent rule applies only when the run builds the agent's image (it is missing, as for a new agent): changing the pin builds nothing, so writing a new pin to every host before the servers are rebuilt, as the upgrade below does, passes. The role reads `k3s --version` in this node's container, and the nodes' kubelet versions and roles through every server of the group in the inventory, each on its own host with its pinned Docker context, so one server stopped or being rebuilt does not hide what the others run. Each server's read, its Docker daemon and `docker exec` included, gets at most `github_runner_cluster_cluster_read_timeout` seconds, so one that hangs is cut off and named rather than holding up every host; the docker process it ran may stay on that host until the daemon answers. When servers run but none of them can be read, it refuses, naming each and why. When no server's container runs it compares only this node's own container and says why: a first bootstrap, a group that joins a cluster whose other servers are outside the inventory (as `bootstrap.sh` builds for a joining server), or a group whose inventory holds no server at all (a host `bootstrap.sh` joins as an agent). `-e github_runner_cluster_allow_k3s_version_jump=true` skips this comparison for one run: for a deliberate downgrade, a cluster that is being replaced, or one that cannot be read. Another server's container name and Docker context are read from its inventory (hostvars), so set them there, not in play vars. + +A new cluster can start on a newer k3s than the default by setting both variables; the integration tests have run only the defaults. + +### Upgrading from a release before the pins + +Releases before the pins built every node from `latest`, and this change refuses what they accepted: + +- `github_runner_cluster_k3s_version: latest`, or `K3S_VERSION=latest` in `.env.bootstrap`: set the `rancher/k3s` tag the cluster runs instead (`k3s --version` in a node's container, with `+` written as `-`), or leave it unset for the default; +- a cluster that floated to a newer k3s minor than the default: set `github_runner_cluster_k3s_version` to the tag it runs, once for the whole group, before running the role; +- a Compose file path (in `github_runner_cluster_dir` or `github_runner_cluster_compose_files`) with a colon in it: move it, since `.env` now lists the files in `COMPOSE_FILE`. + +Set `github_runner_cluster_tailscale_version` too if the nodes run another Tailscale than the default (`tailscale version` in a node's container). The first run after upgrading rewrites each node's `.env` and recreates nothing. + +Changing a pin changes no running node. Compose builds the image only when it is missing and leaves the build arguments out of a container's configuration hash, so the next run of the role rewrites `.env` and leaves the container alone. To move the cluster to a new version, run the role once with the new pins, so every host's `.env` names them, then rebuild and restart the nodes by hand, one at a time. + +The node's `.env` names the Compose files, as absolute paths, and the project the role runs the node with (`COMPOSE_FILE` and `COMPOSE_PROJECT_NAME`), so a `docker compose` command run by hand in `github_runner_cluster_dir`, or with `--project-directory` pointing at it, acts on the same project with the same files, including the extra file a `headscale_in_cluster` bootstrap server or a `github_runner_cluster_compose_files` entry adds. Each command still needs `--profile server` (or `--profile agent` on an agent), since every service sits in a profile, and, on a host with `github_runner_cluster_docker_context` set, that context: run it as `docker --context compose ...`. + +1. Read k3s's release notes for every minor version between the old and the new one. A jump of more than one minor is several upgrades, each run to the end before the next, and the role refuses to build a node more than one minor ahead of the cluster. +2. Servers first, one at a time. Servers run runner pods as well as the control plane, so before each one: + - check that every server's node is Ready (`kubectl get nodes`) and that each server's own API reports etcd healthy (`kubectl get --raw '/readyz?verbose'` through that server lists `[+]etcd ok`), so the cluster keeps its quorum while this one restarts: with three servers, at most one may be down; + - cordon it (`kubectl cordon `) and wait until no runner pod is left on it (`kubectl get pods --all-namespaces --field-selector spec.nodeName=`), so no job is cut off, then drain the rest (`kubectl drain --ignore-daemonsets --delete-emptydir-data`). + + Then, in `github_runner_cluster_dir`, run `docker compose --profile server build k3s` and `docker compose --profile server up -d k3s`. Wait until its node is Ready again and the etcd checks above pass on every server, then `kubectl uncordon `, before the next server. +3. Then agents, one at a time, the same way: cordon, wait for its runner pods to finish, drain, `docker compose --profile agent build k3s-agent` and `docker compose --profile agent up -d k3s-agent`, wait for Ready, uncordon. + +A Tailscale upgrade on its own follows the same order; it restarts each node's mesh client with the node. + +The role copies the new `docker-compose.yml` before it writes `.env`. A run that stops between the two (a failed check, a lost connection) leaves a Compose file that needs `TAILSCALE_VERSION` beside an `.env` without it, and every hand-run `docker compose` command in that directory fails with a missing-variable error until a run of the role succeeds. + ## Docker host disk Each host also runs a `docker-prune` service that removes Docker build cache and dangling images older than `DOCKER_PRUNE_AFTER` (a week by default) once every `DOCKER_PRUNE_INTERVAL_SECONDS` (daily). The node's ephemeral storage lives on the host's Docker disk, shared with anything else that builds there, so an untrimmed cache eventually puts the node under disk pressure and taints it unschedulable. Tagged images, containers and volumes are never touched. Set either variable in the host's `.env` to change them. @@ -95,6 +133,7 @@ Each site is its own cluster: its own inventory group, bootstrap server and mesh - `github_runner_cluster_mesh` (default `tailscale`), `github_runner_cluster_mesh_tag` (default `tag:github-runner-node`), `github_runner_cluster_pod_cidr` (default k3s's `10.42.0.0/16`). - `github_runner_cluster_tailnet_domain`: the mesh's MagicDNS domain. Each node's address is `.`, and join URLs and TLS SANs use it. - `github_runner_cluster_headscale_*`: the Headscale providers' settings, documented in `defaults/main.yml`. +- `github_runner_cluster_k3s_version`, `github_runner_cluster_tailscale_version`: the pinned image tags every node is built from, `github_runner_cluster_allow_k3s_version_jump` (default false) to skip the comparison with what runs, and `github_runner_cluster_cluster_read_timeout` (default 30, a positive whole number of seconds) for each server's read (see [Versions](#versions)). `github_runner_cluster_k3s_tested_version` is the k3s default, which the minor-version check compares hosts that set nothing against; set `github_runner_cluster_k3s_version` instead. - `github_runner_cluster_node_name`: the k3s node name and mesh hostname, defaulting to the inventory name. - `github_runner_cluster_node_taints`: Kubernetes taints (`key=value:Effect`, no spaces) the host's node registers with, empty by default. `NoSchedule` keeps pods without a matching toleration off it and `NoExecute` also evicts the ones already there. k3s applies them on every registration, so they come back if the node object is deleted, which a taint set with `kubectl` does not. - Per-host overrides: `github_runner_cluster_node_role` (`server` or `agent`), `github_runner_cluster_bootstrap` (true picks the bootstrap host, false excludes a host), `github_runner_cluster_server_url` (join this server instead, for a cluster whose servers are outside the inventory), `github_runner_cluster_tls_sans`, `github_runner_cluster_node_address`. The run fails if the result has an even number of servers, no servers, or more than one bootstrap host. diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index a9aaad1..afa370c 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -21,7 +21,15 @@ github_runner_cluster_effective_node_name: "{{ github_runner_cluster_node_name i # Kubernetes taints this host's node registers with, as key=value:Effect strings (for example github-runner.example.com/excluded=true:NoExecute), none containing a space. A taint keeps pods without a matching toleration off the node; NoExecute also evicts the ones already there. They are applied by k3s each time the node registers, so they survive the node object being deleted and registering again, which a taint put on the node by hand does not. Per host, in its host_vars. github_runner_cluster_node_taints: [] -github_runner_cluster_k3s_version: "latest" +# The images every node's k3s container is built from: the rancher/k3s tag (k3s's version with "+" written as "-") and the tailscale/tailscale tag whose tailscale and tailscaled binaries are copied in. Pinned, never latest: Compose builds a node's image only when it is missing, so a floating tag would put whatever version is newest on the day a node is first built, rebuilt or moved into a running cluster, unannounced. Set both once for the whole cluster group, in inventory or group_vars (the check below reads every host's through hostvars). Before it changes anything on a host the role fails if either is not a pinned tag, if the group's hosts name different k3s minor versions, or if the k3s version would break k3s's upgrade path or Kubernetes' version skew policy from what this node and the cluster run now (see the role README's Versions section). Changing them changes no running node: its image is kept until it is rebuilt (see the role README's Versions section for upgrading). +github_runner_cluster_k3s_version: "{{ github_runner_cluster_k3s_tested_version }}" +github_runner_cluster_tailscale_version: "v1.102.4" +# The k3s version this collection's integration tests run, and what github_runner_cluster_k3s_version is for every host that sets none. A separate variable because the minor-version check compares every host of the group, and a host's hostvars never hold role defaults: the check takes this as the version of a host that sets nothing, the same on whichever host runs it. Set github_runner_cluster_k3s_version, never this. +github_runner_cluster_k3s_tested_version: "v1.34.1-k3s1" +# true skips the comparison with what this node and the cluster run, which the role otherwise makes before it changes anything: for a deliberate downgrade, rebuilding a cluster that is being replaced, or a cluster that cannot be read through any server of the inventory. Never needed for an upgrade of one minor at a time. Set it with -e for that run only. +github_runner_cluster_allow_k3s_version_jump: false +# The most seconds the version check spends reading what one server runs, a positive whole number: it bounds the whole read on that server's host (its Docker daemon, the docker exec and the API call, through the task's own timeout) and is also the API call's --request-timeout. Long enough for a busy API server to list the nodes, short enough that a server whose Docker daemon or API server hangs does not hold up every host's run; a server that does not answer in time counts as one that could not be read. 0 is refused, since kubectl reads it as no timeout at all. A read cut off this way can leave its docker process running on that host until the daemon answers. +github_runner_cluster_cluster_read_timeout: 30 # Secrets, as plain variables from any source Ansible reads (ansible-vault, a lookup, or a secrets adapter role such as github_runner_secrets_onepassword). github_runner_cluster_k3s_token is the shared node-join secret and has no default. github_runner_cluster_tailscale_join_key is, with the default tailscale mesh provider, what k3s's own --vpn-auth integration (see docker-compose.yml) uses on every host: it runs Tailscale INSIDE each node's own k3s container, joining the tailnet as its own device and using flannel's native "tailscale" backend for pod traffic. github_runner_cluster_tailscale_api_token, when set, lets tasks/mesh/tailscale_api.yml keep the tailnet's ACL grant and the join key itself provisioned; see that file's comments for what it manages and the one manual step (creating the API token) that has no API to automate. github_runner_cluster_tailscale_join_key: "" diff --git a/roles/github_runner_cluster/files/docker-compose.yml b/roles/github_runner_cluster/files/docker-compose.yml index 0cacc66..6308d12 100644 --- a/roles/github_runner_cluster/files/docker-compose.yml +++ b/roles/github_runner_cluster/files/docker-compose.yml @@ -15,7 +15,8 @@ services: build: context: ./k3s args: - K3S_VERSION: "${K3S_VERSION:-latest}" + K3S_VERSION: "${K3S_VERSION:?K3S_VERSION must be set in .env - the rancher/k3s image tag every node builds from}" + TAILSCALE_VERSION: "${TAILSCALE_VERSION:?TAILSCALE_VERSION must be set in .env - the tailscale/tailscale image tag every node takes its binaries from}" # k3s's own --vpn-auth integration (documented, if experimental) runs Tailscale INSIDE the node's container rather than depending on the host's own Tailscale client: each node's k3s process joins the mesh as its own device, flannel uses its native "tailscale" backend (not VXLAN) for pod traffic, and k3s takes the resulting mesh IP as its node IP. This replaced an earlier, considerably hackier approach (--node-external-ip + --flannel-external-ip layered on the HOST's own Tailscale, tunnelled through Docker's UDP port-publishing) that needed two nested tunnels for the same traffic and was correspondingly fragile. # # k3s/node-entrypoint.sh starts tailscaled, joins the mesh (Tailscale's own control plane, or the Headscale server K3S_VPN_AUTH_CONTROL_SERVER_URL names), then execs k3s; its comments explain each k3s flag (--node-ip, --cluster-init or --server, --tls-san, --resolv-conf) and the live failures behind them. Bind-mounted from this project directory below rather than built into the image: a host that already has the image built would otherwise keep running an old copy, since Compose only builds a missing image. @@ -64,7 +65,8 @@ services: build: context: ./k3s args: - K3S_VERSION: "${K3S_VERSION:-latest}" + K3S_VERSION: "${K3S_VERSION:?K3S_VERSION must be set in .env - the rancher/k3s image tag every node builds from}" + TAILSCALE_VERSION: "${TAILSCALE_VERSION:?TAILSCALE_VERSION must be set in .env - the tailscale/tailscale image tag every node takes its binaries from}" # The same k3s/node-entrypoint.sh as the k3s service; see that service's comment. K3S_URL points at a server's own mesh name, not the host machine's own Tailscale identity. entrypoint: ["/bin/sh", "/usr/local/bin/github-runner-node", "agent"] hostname: ${K3S_AGENT_NODE_NAME:-k3s-agent} diff --git a/roles/github_runner_cluster/files/k3s/Dockerfile b/roles/github_runner_cluster/files/k3s/Dockerfile index e9dcc50..4e0eecd 100644 --- a/roles/github_runner_cluster/files/k3s/Dockerfile +++ b/roles/github_runner_cluster/files/k3s/Dockerfile @@ -5,8 +5,11 @@ # tailscale is copying its official static binaries in directly, confirmed # live: this is exactly what upstream's own docs mean by "installing the # Tailscale service on all nodes" as a prerequisite for the integration. -ARG K3S_VERSION=latest -FROM docker.io/tailscale/tailscale:latest AS tailscale +# +# Both versions are required build arguments with no default, so a build that names neither fails instead of taking whatever latest is that day (the role writes both into .env from github_runner_cluster_k3s_version and github_runner_cluster_tailscale_version). +ARG K3S_VERSION +ARG TAILSCALE_VERSION +FROM docker.io/tailscale/tailscale:${TAILSCALE_VERSION} AS tailscale FROM rancher/k3s:${K3S_VERSION} COPY --from=tailscale /usr/local/bin/tailscaled /usr/local/bin/tailscaled COPY --from=tailscale /usr/local/bin/tailscale /usr/local/bin/tailscale diff --git a/roles/github_runner_cluster/tasks/compose_files.yml b/roles/github_runner_cluster/tasks/compose_files.yml new file mode 100644 index 0000000..740cf3a --- /dev/null +++ b/roles/github_runner_cluster/tasks/compose_files.yml @@ -0,0 +1,12 @@ +--- +# The node's .env lists its Compose files in COMPOSE_FILE as absolute paths separated by colons, for docker compose commands run by hand, so no path may contain a colon. Before the role changes anything on the host. + +- name: Fail if a Compose file path contains a colon + vars: + github_runner_cluster_compose_paths: "{{ github_runner_cluster_compose_file_list | map('regex_replace', '^(?!/)', github_runner_cluster_dir ~ '/') | list }}" + ansible.builtin.fail: + msg: >- + COMPOSE_FILE in the node's .env cannot hold a path with a colon in it, and these Compose files have one (from + github_runner_cluster_dir or github_runner_cluster_compose_files): + {{ github_runner_cluster_compose_paths | select('search', ':') | join(', ') }}. Nothing was changed. + when: github_runner_cluster_compose_paths | select('search', ':') | length > 0 diff --git a/roles/github_runner_cluster/tasks/main.yml b/roles/github_runner_cluster/tasks/main.yml index 92821a2..2d57220 100644 --- a/roles/github_runner_cluster/tasks/main.yml +++ b/roles/github_runner_cluster/tasks/main.yml @@ -34,6 +34,12 @@ - name: Decide this host's part in the cluster ansible.builtin.include_tasks: topology.yml +- name: Check the versions the node's image is built from + ansible.builtin.include_tasks: versions.yml + +- name: Check the k3s version against what this node and the cluster run now + ansible.builtin.include_tasks: running_version.yml + - name: Fail if the k3s token is not set ansible.builtin.fail: msg: >- @@ -42,6 +48,9 @@ such as github_runner_secrets_onepassword first. when: github_runner_cluster_k3s_token | default('') | length == 0 +- name: Check the Compose files the node's .env names + ansible.builtin.include_tasks: compose_files.yml + - name: Create this host's Compose project directory ansible.builtin.file: path: "{{ github_runner_cluster_dir }}/{{ item }}" diff --git a/roles/github_runner_cluster/tasks/running_version.yml b/roles/github_runner_cluster/tasks/running_version.yml new file mode 100644 index 0000000..47d08fd --- /dev/null +++ b/roles/github_runner_cluster/tasks/running_version.yml @@ -0,0 +1,107 @@ +--- +# Compares the k3s version this host's image is to be built from with what already runs: this node's own container, and every node of the cluster as the servers report it. k3s upgrades one minor at a time and never downgrades, so the role refuses what would break k3s's upgrade path or Kubernetes' version skew policy: this node built older than it runs (by patch release and k3s build too), any node more than one minor below the version, a server built an older minor than any other node, or an agent built a newer minor than any server, the last only when the run builds the agent's image (plugins/filter/cluster_versions.py's cluster_k3s_version_change_blockers lists the rules). With github_runner_cluster_allow_k3s_version_jump true nothing here runs. Every task only reads, and runs under --check too; a failure changes nothing on the host. + +- name: Fail if the cluster read timeout is not a positive whole number of seconds + ansible.builtin.fail: + msg: >- + github_runner_cluster_cluster_read_timeout is '{{ github_runner_cluster_cluster_read_timeout }}'; it must be a positive + whole number of seconds (kubectl reads 0 as no timeout at all, which would let one server hold up every host's run). + when: (github_runner_cluster_cluster_read_timeout | string) is not match('^[1-9][0-9]*\\Z') + +- name: Read the k3s version this node's running container reports + # docker ps rather than docker inspect, so that a container that does not exist (a first run) reads as not running while a daemon that cannot be reached fails the run instead of skipping the check. + ansible.builtin.shell: | + set -eu + state=$(docker ps --all --filter "name=^/${container}\$" --format '{{ '{{' }}.State{{ '}}' }}') + if [ "$state" = running ]; then + docker exec "$container" k3s --version + fi + environment: "{{ github_runner_cluster_docker_cli_environment | combine({'container': github_runner_cluster_running_container}) }}" + vars: + github_runner_cluster_running_container: >- + {{ github_runner_cluster_container_name if github_runner_cluster_effective_node_role == 'server' else 'github-runner-k3s-agent' }} + register: github_runner_cluster_running_own + changed_when: false + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + when: not github_runner_cluster_allow_k3s_version_jump | bool + +- name: Read whether this agent's image is already built + # Compose builds the node's image only when it is missing, and only then does the agent's kubelet take the pinned version; while the image exists, changing the pin builds nothing. The image is the one Compose names for the k3s-agent service of this project. + ansible.builtin.command: + argv: [docker, image, ls, --quiet, "{{ github_runner_cluster_compose_project }}-k3s-agent"] + environment: "{{ github_runner_cluster_docker_cli_environment }}" + register: github_runner_cluster_running_agent_image + changed_when: false + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + when: + - not github_runner_cluster_allow_k3s_version_jump | bool + - github_runner_cluster_effective_node_role == 'agent' + +- name: Read every node's k3s version through each server of the inventory + # Through every server of the group, this one first, each on its own host with its own pinned Docker context, since any one of them may be stopped or being rebuilt while the others run. The image's kubectl inside a server's container reads that server's kubeconfig, so the host needs none; it is run as kubectl, since `k3s kubectl` in the rancher/k3s image answers "unknown command". Each read prints "stopped" when the server's container does not run, or "running" and one " " line per node. Each read is bounded by github_runner_cluster_cluster_read_timeout as a whole (the task's own timeout, so a hung Docker daemon or docker exec cannot hold up the run) and as the API call's --request-timeout. A read that fails (a daemon or host that cannot be reached, a read that runs out of time) is recorded rather than failing here, so the filter can say which servers could not be read. Each node line also carries its control-plane label, so the filter can tell servers from agents. + ansible.builtin.shell: | + set -eu + state=$(docker ps --all --filter "name=^/${container}\$" --format '{{ '{{' }}.State{{ '}}' }}') + if [ "$state" != running ]; then + echo stopped + exit 0 + fi + echo running + docker exec "$container" kubectl get nodes --request-timeout="${timeout}s" \ + -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{.status.nodeInfo.kubeletVersion}{" "}{.metadata.labels.node-role\.kubernetes\.io/control-plane}{"\n"}{end}' + environment: >- + {{ ({'DOCKER_CONTEXT': github_runner_cluster_running_read_context} if github_runner_cluster_running_read_context | length > 0 else {}) + | combine({'container': github_runner_cluster_running_read_container, 'timeout': github_runner_cluster_cluster_read_timeout | string}) }} + delegate_to: "{{ item }}" + loop: "{{ github_runner_cluster_running_servers }}" + vars: + github_runner_cluster_running_servers: >- + {{ ([inventory_hostname] if github_runner_cluster_effective_node_role == 'server' else []) + + (github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') + | map(attribute='key') | reject('equalto', inventory_hostname) | list) }} + github_runner_cluster_running_read_container: >- + {{ github_runner_cluster_container_name if item == inventory_hostname + else hostvars[item].github_runner_cluster_container_name | default('github-runner-k3s') }} + github_runner_cluster_running_read_context: >- + {{ (github_runner_cluster_docker_context if item == inventory_hostname + else hostvars[item].github_runner_cluster_docker_context | default('')) | trim }} + register: github_runner_cluster_running_reads + changed_when: false + failed_when: false + ignore_unreachable: true + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + when: not github_runner_cluster_allow_k3s_version_jump | bool + +- name: Say what the version check could not compare + # Only when no server answered and none failed: every server of the inventory was read as stopped, or there is none. Which of those it is, and whether the cluster lives outside the inventory, comes from the topology: a group with no bootstrap host joins a cluster that one of its hosts did not bootstrap. + vars: + github_runner_cluster_running_joins_outside: >- + {{ github_runner_cluster_topology.hosts | dict2items | selectattr('value.bootstrap') | list | length == 0 }} + ansible.builtin.debug: + msg: >- + {% if github_runner_cluster_running_reads.results | default([]) | length == 0 %}No server of {{ github_runner_cluster_group }} is in the inventory, so the cluster this node joins cannot be read; only this node's own container was compared.{% elif github_runner_cluster_running_joins_outside | bool %}{{ github_runner_cluster_group }} joins a cluster whose other servers are outside the inventory, and none of its own servers runs its container, so the cluster could not be read; only this node's own container was compared.{% else %}No server of {{ github_runner_cluster_group }} runs its container, so there is no cluster to compare with yet (a first bootstrap, or a cluster whose servers are all stopped); only this node's own container was compared.{% endif %} + when: + - not github_runner_cluster_allow_k3s_version_jump | bool + - github_runner_cluster_running_reads.results | default([]) | rejectattr('stdout', 'defined') | list | length == 0 + - github_runner_cluster_running_reads.results | default([]) | map(attribute='stdout') | map('trim') | reject('equalto', 'stopped') | list | length == 0 + +- name: Refuse a k3s version outside the upgrade path from what runs now + vars: + github_runner_cluster_version_change_blockers: >- + {{ github_runner_cluster_k3s_version | exadev.github_runner.cluster_k3s_version_change_blockers( + github_runner_cluster_effective_node_name, github_runner_cluster_running_own.stdout, + github_runner_cluster_running_reads.results | default([]), github_runner_cluster_effective_node_role, + github_runner_cluster_running_agent_image.stdout | default('') | trim | length == 0) }} + ansible.builtin.fail: + msg: >- + Not building {{ github_runner_cluster_effective_node_name }} from k3s {{ github_runner_cluster_k3s_version }}: + {{ github_runner_cluster_version_change_blockers | join(' ') }} + Set github_runner_cluster_k3s_version to the minor the cluster runs, or one minor above it to upgrade (see the role + README's Versions section). To build it anyway, for a deliberate downgrade, a cluster being replaced or one that cannot + be read, run once with -e github_runner_cluster_allow_k3s_version_jump=true. Nothing was changed. + when: + - not github_runner_cluster_allow_k3s_version_jump | bool + - github_runner_cluster_version_change_blockers | length > 0 diff --git a/roles/github_runner_cluster/tasks/versions.yml b/roles/github_runner_cluster/tasks/versions.yml new file mode 100644 index 0000000..c3d07dd --- /dev/null +++ b/roles/github_runner_cluster/tasks/versions.yml @@ -0,0 +1,21 @@ +--- +# Checks the image versions every node is built from (github_runner_cluster_k3s_version, github_runner_cluster_tailscale_version) before the role changes anything on the host: each must be a pinned tag, and every host of the cluster group must name the same k3s minor version. + +- name: Fail if the node's image versions are not pinned + vars: + github_runner_cluster_version_errors: >- + {{ github_runner_cluster_k3s_version | exadev.github_runner.cluster_version_errors(github_runner_cluster_tailscale_version) }} + ansible.builtin.fail: + msg: "{{ github_runner_cluster_version_errors | join(' ') }}" + when: github_runner_cluster_version_errors | length > 0 + +- name: Fail if the cluster's hosts name different k3s minor versions + # Every host is compared through hostvars, its own included, and a host that sets nothing counts as the role's tested version, so the result is the same on whichever host runs it, and under --limit to the host that is out of step. Before any node starts: a node built from another minor joins the running cluster's etcd and control plane outside the one-minor-at-a-time upgrade path. Upgrading means changing the version for the whole group and rebuilding nodes one by one (see the role README). + vars: + github_runner_cluster_version_skew: >- + {{ groups[github_runner_cluster_group] | exadev.github_runner.cluster_k3s_minor_skew(hostvars, github_runner_cluster_k3s_tested_version) }} + ansible.builtin.fail: + msg: >- + The hosts of {{ github_runner_cluster_group }} do not all name the same k3s minor version + ({{ github_runner_cluster_version_skew | join(', ') }}). Set github_runner_cluster_k3s_version once for the whole group. + when: github_runner_cluster_version_skew | length > 0 diff --git a/roles/github_runner_cluster/templates/env-agent.j2 b/roles/github_runner_cluster/templates/env-agent.j2 index 8f5b742..04ab1e2 100644 --- a/roles/github_runner_cluster/templates/env-agent.j2 +++ b/roles/github_runner_cluster/templates/env-agent.j2 @@ -1,6 +1,10 @@ # Generated by the exadev.github_runner collection's github_runner_cluster role on every run. Do not edit by hand: change this host's ansible/host_vars file and rerun ansible-playbook, or, on a host brought up with bootstrap.sh, change .env.bootstrap and rerun ./bootstrap.sh. +# The Compose files, as absolute paths, and the project the role runs this node with, so that a docker compose command run by hand in this directory, or with --project-directory from another, acts on the same project with the same files (each command still needs its --profile, and the host's pinned Docker context when it has one). +COMPOSE_FILE={{ github_runner_cluster_compose_file_list | map('regex_replace', '^(?!/)', github_runner_cluster_dir ~ '/') | join(':') }} +COMPOSE_PROJECT_NAME={{ github_runner_cluster_compose_project }} K3S_TOKEN={{ github_runner_cluster_k3s_token }} K3S_VERSION={{ github_runner_cluster_k3s_version }} +TAILSCALE_VERSION={{ github_runner_cluster_tailscale_version }} K3S_AGENT_SERVER_URL={{ github_runner_cluster_effective_server_url }} K3S_AGENT_NODE_NAME={{ github_runner_cluster_effective_node_name }} K3S_VPN_AUTH_JOIN_KEY={{ github_runner_cluster_mesh_join_key }} diff --git a/roles/github_runner_cluster/templates/env.j2 b/roles/github_runner_cluster/templates/env.j2 index 27314ff..71b7f47 100644 --- a/roles/github_runner_cluster/templates/env.j2 +++ b/roles/github_runner_cluster/templates/env.j2 @@ -1,6 +1,10 @@ # Generated by the exadev.github_runner collection's github_runner_cluster role on every run. Do not edit by hand: change this host's ansible/host_vars file and rerun ansible-playbook, or, on a host brought up with bootstrap.sh, change .env.bootstrap and rerun ./bootstrap.sh. +# The Compose files, as absolute paths, and the project the role runs this node with, so that a docker compose command run by hand in this directory, or with --project-directory from another, acts on the same project with the same files (each command still needs its --profile, and the host's pinned Docker context when it has one). +COMPOSE_FILE={{ github_runner_cluster_compose_file_list | map('regex_replace', '^(?!/)', github_runner_cluster_dir ~ '/') | join(':') }} +COMPOSE_PROJECT_NAME={{ github_runner_cluster_compose_project }} K3S_TOKEN={{ github_runner_cluster_k3s_token }} K3S_VERSION={{ github_runner_cluster_k3s_version }} +TAILSCALE_VERSION={{ github_runner_cluster_tailscale_version }} K3S_TLS_SAN_LIST={{ github_runner_cluster_effective_tls_sans | join(' ') }} K3S_VPN_AUTH_JOIN_KEY={{ github_runner_cluster_mesh_join_key }} {% if github_runner_cluster_mesh_control_server_url | length > 0 %} diff --git a/tests/cluster_versions/checks.yml b/tests/cluster_versions/checks.yml new file mode 100644 index 0000000..4b4e096 --- /dev/null +++ b/tests/cluster_versions/checks.yml @@ -0,0 +1,12 @@ +--- +# Runs some of the cluster role's task files (check_tasks, comma-separated: versions, the default, running_version, compose_files, or topology followed by another) through the role itself, for tests/unit/test_cluster_versions.py, so that its defaults load as role defaults do in a real run: below the inventory the test writes, and absent from every host's hostvars. +- name: Run the cluster role's version checks + hosts: "{{ check_hosts | default('github_runner_cluster') }}" + connection: local + gather_facts: false + tasks: + - name: Include the role's checks + ansible.builtin.include_role: + name: exadev.github_runner.github_runner_cluster + tasks_from: "{{ item }}" + loop: "{{ (check_tasks | default('versions')).split(',') }}" diff --git a/tests/cluster_versions/render_env.yml b/tests/cluster_versions/render_env.yml new file mode 100644 index 0000000..a806eab --- /dev/null +++ b/tests/cluster_versions/render_env.yml @@ -0,0 +1,27 @@ +--- +# Renders the cluster role's env.j2 and env-agent.j2 with the role's defaults and placeholder values for the settings this test does not look at, writing them to env_output and env_agent_output, for tests/unit/test_cluster_versions.py. Role variables are passed as extra variables, which outrank the defaults loaded here. +- name: Render the node .env templates + hosts: localhost + connection: local + gather_facts: false + vars_files: + - ../../roles/github_runner_cluster/defaults/main.yml + vars: + github_runner_cluster_k3s_token: example-token + github_runner_cluster_effective_tls_sans: [node1.example.internal] + github_runner_cluster_mesh_join_key: example-key + github_runner_cluster_mesh_control_server_url: "" + github_runner_cluster_effective_server_url: "" + github_runner_cluster_effective_node_name: node1 + tasks: + - name: Render env.j2 + ansible.builtin.copy: + content: "{{ lookup('ansible.builtin.template', playbook_dir ~ '/../../roles/github_runner_cluster/templates/env.j2') }}" + dest: "{{ env_output }}" + mode: "0600" + + - name: Render env-agent.j2 + ansible.builtin.copy: + content: "{{ lookup('ansible.builtin.template', playbook_dir ~ '/../../roles/github_runner_cluster/templates/env-agent.j2') }}" + dest: "{{ env_agent_output }}" + mode: "0600" diff --git a/tests/unit/test_cluster_versions.py b/tests/unit/test_cluster_versions.py new file mode 100644 index 0000000..23b3393 --- /dev/null +++ b/tests/unit/test_cluster_versions.py @@ -0,0 +1,686 @@ +"""Tests for how the cluster role pins the versions every node's k3s image is built from: the filters (plugins/filter/cluster_versions.py), the checks in tasks/versions.yml and tasks/running_version.yml (run through the role with tests/cluster_versions/checks.yml), the .env the role writes (tests/cluster_versions/render_env.yml), the Compose file and Dockerfile that consume it, and bootstrap.sh's mapping of .env.bootstrap onto the role's variables.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import re +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ROLE = ROOT / "roles" / "github_runner_cluster" +PLUGIN = ROOT / "plugins" / "filter" / "cluster_versions.py" +_spec = importlib.util.spec_from_file_location("cluster_versions_filters", PLUGIN) +assert _spec is not None and _spec.loader is not None +versions = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(versions) + +ANSIBLE_PLAYBOOK = os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook") +DEFAULTS: dict[str, Any] = yaml.safe_load((ROLE / "defaults" / "main.yml").read_text()) +COMPOSE = ROLE / "files" / "docker-compose.yml" +DOCKERFILE = ROLE / "files" / "k3s" / "Dockerfile" + + +def ansible_env() -> dict[str, str]: + return {**os.environ, "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), "ANSIBLE_LOCALHOST_WARNING": "false", "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", "ANSIBLE_RETRY_FILES_ENABLED": "false"} + + +TESTED = DEFAULTS["github_runner_cluster_k3s_tested_version"] + + +class DefaultsTest(unittest.TestCase): + def test_the_defaults_are_pinned_tags(self) -> None: + self.assertEqual(versions.cluster_version_errors(TESTED, DEFAULTS["github_runner_cluster_tailscale_version"]), []) + + def test_the_k3s_version_defaults_to_the_tested_one(self) -> None: + self.assertEqual(DEFAULTS["github_runner_cluster_k3s_version"], "{{ github_runner_cluster_k3s_tested_version }}") + self.assertIs(DEFAULTS["github_runner_cluster_allow_k3s_version_jump"], False) + + +class VersionErrorsTest(unittest.TestCase): + def test_pinned_tags_pass(self) -> None: + self.assertEqual(versions.cluster_version_errors("v1.34.1-k3s1", "v1.102.4"), []) + + def test_latest_and_other_floating_or_malformed_values_fail(self) -> None: + for k3s in ("latest", "", None, "v1.34", "v1.34.1+k3s1", "1.34.1-k3s1", "stable"): + with self.subTest(k3s=k3s): + self.assertIn("github_runner_cluster_k3s_version", " ".join(versions.cluster_version_errors(k3s, "v1.102.4"))) + for tailscale in ("latest", "stable", "", "1.102.4", "v1.102", None): + with self.subTest(tailscale=tailscale): + self.assertIn("github_runner_cluster_tailscale_version", " ".join(versions.cluster_version_errors("v1.34.1-k3s1", tailscale))) + + def test_both_problems_are_reported(self) -> None: + self.assertEqual(len(versions.cluster_version_errors("latest", "latest")), 2) + + +class MinorSkewTest(unittest.TestCase): + def test_one_version_everywhere_is_no_skew(self) -> None: + self.assertEqual(versions.cluster_k3s_minor_skew(["a", "b"], {"a": {}, "b": {}}, "v1.34.1-k3s1"), []) + + def test_patch_and_build_differences_within_a_minor_are_no_skew(self) -> None: + hostvars = {"a": {"github_runner_cluster_k3s_version": "v1.34.1-k3s1"}, "b": {"github_runner_cluster_k3s_version": "v1.34.6-k3s2"}} + self.assertEqual(versions.cluster_k3s_minor_skew(["a", "b"], hostvars, "v1.34.1-k3s1"), []) + + def test_another_minor_on_one_host_lists_every_host(self) -> None: + hostvars = {"a": {}, "b": {"github_runner_cluster_k3s_version": "v1.35.0-k3s1"}, "c": {}} + self.assertEqual(versions.cluster_k3s_minor_skew(["a", "b", "c"], hostvars, "v1.34.1-k3s1"), ["a: v1.34.1-k3s1", "b: v1.35.0-k3s1", "c: v1.34.1-k3s1"]) + + def test_another_major_with_the_same_minor_number_is_skew(self) -> None: + hostvars = {"a": {"github_runner_cluster_k3s_version": "v2.34.0-k3s1"}, "b": {}} + self.assertNotEqual(versions.cluster_k3s_minor_skew(["a", "b"], hostvars, "v1.34.1-k3s1"), []) + + def test_unusable_input_is_an_error(self) -> None: + with self.assertRaises(versions.AnsibleFilterError): + versions.cluster_k3s_minor_skew("a", {}, "v1.34.1-k3s1") + + def test_the_filters_are_registered(self) -> None: + filters = versions.FilterModule().filters() + self.assertIs(filters["cluster_version_errors"], versions.cluster_version_errors) + self.assertIs(filters["cluster_k3s_minor_skew"], versions.cluster_k3s_minor_skew) + self.assertIs(filters["cluster_k3s_version_change_blockers"], versions.cluster_k3s_version_change_blockers) + + +def answered(host: str, nodes: str) -> dict[str, Any]: + """A server's read of the cluster that answered, as the role's loop registers it.""" + return {"item": host, "rc": 0, "stdout": "running\n" + nodes, "stderr": ""} + + +def stopped(host: str) -> dict[str, Any]: + return {"item": host, "rc": 0, "stdout": "stopped", "stderr": ""} + + +def failed(host: str, reason: str = "The connection to the server 127.0.0.1:6443 was refused") -> dict[str, Any]: + return {"item": host, "rc": 1, "stdout": "running", "stderr": reason} + + +class VersionChangeBlockersTest(unittest.TestCase): + OWN = "k3s version v1.34.1+k3s1 (24fc436e)\n" + + def blockers(self, pin: str, own: str = "", reads: list[dict[str, Any]] | None = None) -> list[str]: + result: list[str] = versions.cluster_k3s_version_change_blockers(pin, "node1", own, reads or []) + return result + + def test_nothing_running_has_nothing_to_compare(self) -> None: + self.assertEqual(self.blockers("v1.40.0-k3s1"), []) + self.assertEqual(self.blockers("v1.40.0-k3s1", reads=[stopped("a"), stopped("b")]), []) + + def test_the_same_minor_and_one_minor_up_pass(self) -> None: + for pin in ("v1.34.1-k3s1", "v1.34.6-k3s2", "v1.35.0-k3s1"): + with self.subTest(pin=pin): + self.assertEqual(self.blockers(pin, self.OWN, [answered("a", "node1 v1.34.1+k3s1\nnode2 v1.34.1+k3s1\n")]), []) + + def test_an_older_minor_than_the_container_runs_is_refused(self) -> None: + self.assertEqual(self.blockers("v1.33.5-k3s1", self.OWN), ["This node's own container (node1) runs v1.34.1+k3s1, newer than v1.33.5-k3s1; k3s cannot be downgraded."]) + + def test_two_minors_above_a_node_is_refused(self) -> None: + self.assertEqual(self.blockers("v1.36.0-k3s1", reads=[answered("a", "node2 v1.34.1+k3s1\nnode3 v1.35.2+k3s1")]), ["Node node2 runs v1.34.1+k3s1, more than one minor away from v1.36.0-k3s1; k3s upgrades the cluster one minor at a time."]) + + def test_another_major_is_refused(self) -> None: + for pin in ("v2.35.0-k3s1", "v0.36.0-k3s1"): + with self.subTest(pin=pin): + self.assertEqual(len(self.blockers(pin, reads=[answered("a", "node2 v1.34.1+k3s1")])), 1) + + def test_a_version_that_cannot_be_read_is_refused(self) -> None: + self.assertEqual(self.blockers("v1.34.1-k3s1", reads=[answered("a", "node2 ")]), ["Node node2 reports '', which is not a k3s version this check can read."]) + + def test_every_server_that_answers_counts(self) -> None: + reads = [stopped("a"), answered("b", "node2 v1.36.0+k3s1 true"), answered("c", "node3 v1.36.0+k3s1")] + self.assertEqual(self.blockers("v1.34.1-k3s1", reads=reads), ["Server node2 runs v1.36.0+k3s1, a newer minor than v1.34.1-k3s1; the servers cannot go back a minor.", "Agent node3 runs v1.36.0+k3s1, a newer minor than v1.34.1-k3s1; a kubelet may not be newer than the API servers."]) + + def test_one_server_that_answers_is_enough(self) -> None: + self.assertEqual(self.blockers("v1.34.1-k3s1", reads=[failed("a"), {"item": "b", "unreachable": True, "msg": "ssh: connect"}, answered("c", "node1 v1.34.1+k3s1")]), []) + + def test_a_cluster_that_no_server_can_be_read_through_is_refused(self) -> None: + found = self.blockers("v1.34.1-k3s1", reads=[failed("a"), stopped("b"), {"item": "c", "unreachable": True, "msg": "ssh: connect to host c"}]) + self.assertEqual(found, ["The cluster could not be read through any server of the inventory, so what it runs cannot be told: a could not be read (The connection to the server 127.0.0.1:6443 was refused); c could not be reached (ssh: connect to host c)."]) + + def test_an_answer_that_cannot_be_read_counts_as_unread(self) -> None: + self.assertIn("gave an answer this check cannot read", self.blockers("v1.34.1-k3s1", reads=[{"item": "a", "rc": 0, "stdout": "", "stderr": ""}])[0]) + + def test_a_patch_or_build_downgrade_is_refused(self) -> None: + for pin in ("v1.34.0-k3s1", "v1.34.1-k3s0"): + with self.subTest(pin=pin): + self.assertEqual(self.blockers(pin, self.OWN), [f"This node's own container (node1) runs v1.34.1+k3s1, newer than {pin}; k3s cannot be downgraded."]) + self.assertEqual(self.blockers("v1.34.1-k3s2", self.OWN), []) + + def test_a_downgrade_to_an_older_major_is_refused_as_a_downgrade(self) -> None: + self.assertEqual(self.blockers("v1.40.0-k3s1", reads=[answered("a", "node2 v2.1.0+k3s1 true")]), ["Node node2 runs v2.1.0+k3s1, more than one minor away from v1.40.0-k3s1; k3s upgrades the cluster one minor at a time."]) + + def test_a_node_two_servers_report_is_named_once(self) -> None: + reads = [answered("a", "node2 v1.36.0+k3s1 true"), answered("b", "node2 v1.36.0+k3s1 true")] + self.assertEqual(len(self.blockers("v1.34.1-k3s1", reads=reads)), 1) + + def test_an_agent_may_not_move_above_any_server(self) -> None: + reads = [answered("a", "server1 v1.34.1+k3s1 true\nserver2 v1.35.0+k3s1 true\nagent1 v1.34.1+k3s1")] + found = versions.cluster_k3s_version_change_blockers("v1.35.0-k3s1", "agent1", "", reads, "agent") + self.assertEqual(found, ["Server server1 runs v1.34.1+k3s1, an older minor than v1.35.0-k3s1; an agent's kubelet may not be newer than the API servers, so upgrade the servers first."]) + self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.35.0-k3s1", "server3", "", reads, "server"), []) + upgraded = [answered("a", "server1 v1.35.0+k3s1 true\nserver2 v1.35.0+k3s1 true\nagent1 v1.34.1+k3s1")] + self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.35.0-k3s1", "agent1", "k3s version v1.34.1+k3s1 (x)", upgraded, "agent"), []) + self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.35.1-k3s1", "agent1", "", upgraded, "agent"), []) + + def test_an_agent_whose_image_is_already_built_may_take_a_pin_above_the_servers(self) -> None: + reads = [answered("a", "server1 v1.34.1+k3s1 true\nagent1 v1.34.1+k3s1")] + self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.35.0-k3s1", "agent1", "k3s version v1.34.1+k3s1 (x)", reads, "agent", False), []) + + def test_a_timed_out_read_counts_as_unread_on_every_ansible_core(self) -> None: + for result in ( + {"item": "a", "failed": True, "timedout": {"period": 2}, "msg": "Task failed: Timed out after 2 second(s)."}, + {"item": "a", "failed": True, "msg": "The ansible.builtin.shell action failed to execute in the expected time frame (2) and was terminated"}, + ): + with self.subTest(result=result): + self.assertIn("a did not answer within github_runner_cluster_cluster_read_timeout", self.blockers("v1.34.1-k3s1", reads=[result])[0]) + + def test_a_failure_with_only_a_message_names_it(self) -> None: + found = self.blockers("v1.34.1-k3s1", reads=[{"item": "a", "failed": True, "msg": "docker: command not found"}]) + self.assertIn("a could not be read (docker: command not found)", found[0]) + + def test_patch_releases_of_other_nodes_are_not_compared(self) -> None: + reads = [answered("a", "server1 v1.35.0+k3s1 true\nagent1 v1.35.1+k3s1")] + self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.35.0-k3s1", "server1", "", reads, "server"), []) + servers = [answered("a", "server1 v1.35.1+k3s1 true")] + self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.35.0-k3s1", "agent1", "", servers, "agent"), []) + self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.35.2-k3s1", "agent1", "", servers, "agent"), []) + + def test_a_server_may_not_be_built_below_a_node_s_minor(self) -> None: + reads = [answered("a", "server1 v1.34.1+k3s1 true\nagent1 v1.35.0+k3s1")] + self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.34.1-k3s1", "server1", "", reads, "server"), ["Agent agent1 runs v1.35.0+k3s1, a newer minor than v1.34.1-k3s1; a kubelet may not be newer than the API servers."]) + + def test_this_node_s_own_entry_in_the_node_list_counts_as_this_node(self) -> None: + reads = [answered("a", "server1 v1.35.0+k3s1 true\nserver2 v1.35.0+k3s1 true")] + self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.34.9-k3s1", "server1", "", reads, "server")[0], "Node server1 runs v1.35.0+k3s1, newer than v1.34.9-k3s1; k3s cannot be downgraded.") + + def test_unusable_input_is_an_error(self) -> None: + with self.assertRaises(versions.AnsibleFilterError): + versions.cluster_k3s_version_change_blockers("v1.34.1-k3s1", "node1", "", [], "worker") + with self.assertRaises(versions.AnsibleFilterError): + self.blockers("latest") + with self.assertRaises(versions.AnsibleFilterError): + versions.cluster_k3s_version_change_blockers("v1.34.1-k3s1", "node1", None, []) + with self.assertRaises(versions.AnsibleFilterError): + versions.cluster_k3s_version_change_blockers("v1.34.1-k3s1", "node1", "", ["not a result"]) + + +class TaskOrderTest(unittest.TestCase): + def test_the_role_checks_the_versions_before_it_changes_the_host(self) -> None: + tasks = yaml.safe_load((ROLE / "tasks" / "main.yml").read_text()) + includes = [task.get("ansible.builtin.include_tasks") for task in tasks] + first_change = next(index for index, task in enumerate(tasks) if "ansible.builtin.file" in task or "ansible.builtin.copy" in task or "block" in task) + for checks in ("versions.yml", "running_version.yml"): + with self.subTest(checks=checks): + self.assertIn(checks, includes) + self.assertLess(includes.index(checks), first_change) + + +class DockerfileTest(unittest.TestCase): + def test_both_versions_are_build_arguments_without_a_default(self) -> None: + lines = DOCKERFILE.read_text().splitlines() + args = [line.split()[1] for line in lines if line.startswith("ARG ")] + self.assertEqual(sorted(args), ["K3S_VERSION", "TAILSCALE_VERSION"], "an ARG with a default (NAME=value) would let a build float") + + def test_every_base_image_is_taken_from_a_build_argument(self) -> None: + bases = [line.split()[1] for line in DOCKERFILE.read_text().splitlines() if line.startswith("FROM ")] + self.assertEqual(bases, ["docker.io/tailscale/tailscale:${TAILSCALE_VERSION}", "rancher/k3s:${K3S_VERSION}"]) + self.assertNotIn(":latest", DOCKERFILE.read_text()) + + +class ComposeFileTest(unittest.TestCase): + def test_every_service_built_from_the_k3s_context_requires_both_versions(self) -> None: + # Compose stops at the first unset required variable in the whole file, so the render test below cannot tell whether a second service would also refuse; this reads each service's own arguments. + services = yaml.safe_load(COMPOSE.read_text())["services"] + built = {name: service["build"]["args"] for name, service in services.items() if "build" in service} + self.assertEqual(sorted(built), ["k3s", "k3s-agent"]) + for name, args in built.items(): + for variable in ("K3S_VERSION", "TAILSCALE_VERSION"): + with self.subTest(service=name, variable=variable): + self.assertRegex(args[variable], rf"^\$\{{{variable}:\?[^}}]+\}}$") + + +def has_compose() -> bool: + if shutil.which("docker") is None: + return False + return subprocess.run(["docker", "compose", "version"], capture_output=True, check=False).returncode == 0 + + +@unittest.skipUnless(has_compose(), "docker compose is not installed") +class ComposeTest(unittest.TestCase): + def render(self, **env: str) -> subprocess.CompletedProcess[str]: + clean = {key: value for key, value in os.environ.items() if not key.startswith(("DOCKER_", "COMPOSE_", "K3S_", "TAILSCALE_"))} + args = ["docker", "compose", "--project-directory", tempfile.gettempdir(), "--project-name", "example", "-f", str(COMPOSE), "--profile", "server", "--profile", "agent", "config", "--format", "json"] + return subprocess.run(args, capture_output=True, text=True, check=False, env=clean | {"K3S_TOKEN": "t", "K3S_TLS_SAN_LIST": "n"} | env) + + def test_both_node_services_build_from_the_pinned_versions(self) -> None: + result = self.render(K3S_VERSION="v1.34.1-k3s1", TAILSCALE_VERSION="v1.102.4") + self.assertEqual(result.returncode, 0, result.stderr) + services = json.loads(result.stdout)["services"] + for name in ("k3s", "k3s-agent"): + with self.subTest(service=name): + self.assertEqual(services[name]["build"]["args"], {"K3S_VERSION": "v1.34.1-k3s1", "TAILSCALE_VERSION": "v1.102.4"}) + + def test_a_missing_version_refuses_to_render(self) -> None: + for env, missing in (({"TAILSCALE_VERSION": "v1.102.4"}, "K3S_VERSION"), ({"K3S_VERSION": "v1.34.1-k3s1"}, "TAILSCALE_VERSION")): + with self.subTest(missing=missing): + result = self.render(**env) + self.assertNotEqual(result.returncode, 0) + self.assertIn(f"{missing} must be set in .env", result.stderr) + + +# Stands in for the docker CLI in the running-version checks: answers `docker ps --all --filter name=^/$ --format ...` and `docker exec k3s --version` or `kubectl get nodes` from files in $DOCKER_STUB (.state, .version, .nodes, .kubectl-fails), fails every call when $DOCKER_STUB/daemon-down exists, or daemon-down- for that DOCKER_CONTEXT, sleeps for the seconds hang- holds before answering, answers `docker image ls --quiet ` from .image, and logs each call with its DOCKER_CONTEXT. +DOCKER_STUB = """#!/usr/bin/env python3 +import os, re, sys, time +stub = os.environ["DOCKER_STUB"] +args = sys.argv[1:] +with open(os.path.join(stub, "calls.log"), "a") as log: + log.write(os.environ.get("DOCKER_CONTEXT", "-") + " " + " ".join(args) + "\\n") +def read(name): + path = os.path.join(stub, name) + return open(path).read() if os.path.exists(path) else None +if read("daemon-down") is not None or read("daemon-down-" + os.environ.get("DOCKER_CONTEXT", "-")) is not None: + sys.exit("Cannot connect to the Docker daemon") +if read("hang-" + os.environ.get("DOCKER_CONTEXT", "-")) is not None: + time.sleep(int(read("hang-" + os.environ.get("DOCKER_CONTEXT", "-")))) +if args[0] == "ps": + container = re.match(r"name=\\^/(.*)\\$$", args[args.index("--filter") + 1]).group(1) + print(read(container + ".state") or "", end="") +elif args[:3] == ["image", "ls", "--quiet"]: + print("sha256:0123" if read(args[3] + ".image") is not None else "") +elif args[0] == "exec" and args[2:4] == ["k3s", "--version"]: + print(read(args[1] + ".version")) +elif args[0] == "exec" and args[2:4] == ["kubectl", "get"]: + if read(args[1] + ".kubectl-fails") is not None: + sys.exit("The connection to the server 127.0.0.1:6443 was refused") + print(read(args[1] + ".nodes"), end="") +else: + sys.exit("unexpected docker call: " + " ".join(args)) +""" + + +def failed_task(stdout: str) -> str | None: + """Return the name of the task whose header last precedes a "fatal:" line in ansible-playbook's output, or None when nothing failed.""" + task = None + for line in stdout.splitlines(): + header = re.match(r"TASK \[(?:[^:\]]+ : )?(.*)\] \*+$", line) + if header: + task = header.group(1) + elif line.startswith("fatal: "): + return task + return None + + +@unittest.skipIf(shutil.which(ANSIBLE_PLAYBOOK) is None, "ansible-playbook is not installed") +class RoleTest(unittest.TestCase): + def setUp(self) -> None: + self.work = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.work) + self.stub = self.work / "stub" + (self.work / "bin").mkdir() + self.stub.mkdir() + docker = self.work / "bin" / "docker" + docker.write_text(DOCKER_STUB) + docker.chmod(0o755) + + def run_checks(self, hosts: dict[str, dict[str, Any]], *extra: str, group_vars: dict[str, Any] | None = None, check_tasks: str = "versions", timeout: int | None = None) -> subprocess.CompletedProcess[str]: + inventory = self.work / "inventory.yml" + inventory.write_text(yaml.safe_dump({"all": {"children": {"github_runner_cluster": {"vars": group_vars or {}, "hosts": hosts}}}})) + args = [ANSIBLE_PLAYBOOK, "-i", str(inventory), str(ROOT / "tests" / "cluster_versions" / "checks.yml"), "-e", f"check_tasks={check_tasks}", *extra] + env = ansible_env() | {"PATH": f"{self.work / 'bin'}{os.pathsep}{os.environ['PATH']}", "DOCKER_STUB": str(self.stub)} + return subprocess.run(args, capture_output=True, text=True, check=False, env=env, cwd=self.work, timeout=timeout) + + def skew(self, host_versions: dict[str, str | None], *extra: str) -> subprocess.CompletedProcess[str]: + hosts = {name: ({"github_runner_cluster_k3s_version": version} if version else {}) for name, version in host_versions.items()} + return self.run_checks(hosts, *extra) + + def test_the_defaults_everywhere_pass(self) -> None: + result = self.skew({"node1": None, "node2": None, "node3": None}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_one_version_for_the_whole_group_passes(self) -> None: + result = self.run_checks({"node1": {}, "node2": {}}, group_vars={"github_runner_cluster_k3s_version": "v1.35.0-k3s1"}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_latest_fails_before_anything_else(self) -> None: + result = self.skew({"node1": None}, "-e", "github_runner_cluster_k3s_version=latest") + self.assertNotEqual(result.returncode, 0) + self.assertIn("never latest", result.stdout) + + def test_a_host_on_another_minor_fails_every_host_including_itself(self) -> None: + result = self.skew({"node1": "v1.36.0-k3s1", "node2": None, "node3": None}) + self.assertNotEqual(result.returncode, 0) + self.assertIn(f"node1: v1.36.0-k3s1, node2: {TESTED}, node3: {TESTED}", result.stdout) + self.assertEqual(sorted(re.findall(r"fatal: \[(node\d)\]", result.stdout)), ["node1", "node2", "node3"]) + + def test_the_host_on_another_minor_fails_when_run_on_its_own(self) -> None: + result = self.skew({"node1": "v1.36.0-k3s1", "node2": None, "node3": None}, "--limit", "node1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("fatal: [node1]", result.stdout) + + def running(self, *extra: str, pin: str | None = None, run_on: str = "node2", roles: dict[str, str] | None = None, bootstrap: str | None = "node1", unreachable: str | None = None, timeout: int | None = None) -> subprocess.CompletedProcess[str]: + """Run tasks/running_version.yml on one host of a three-host group (servers unless ``roles`` says otherwise) whose topology names ``bootstrap`` as its bootstrap host, or none; ``unreachable`` names a host reached over SSH on a port nothing listens on.""" + roles = {"node1": "server", "node2": "server", "node3": "server", **(roles or {})} + topology = {"hosts": {name: {"role": role, "bootstrap": name == bootstrap, "node_name": name} for name, role in roles.items()}} + common: dict[str, Any] = {"github_runner_cluster_topology": topology, **({"github_runner_cluster_k3s_version": pin} if pin else {})} + hosts = { + name: { + "github_runner_cluster_effective_node_role": role, + "github_runner_cluster_effective_node_name": name, + "github_runner_cluster_container_name": f"{name}-k3s", + "github_runner_cluster_docker_context": f"ctx-{name}", + "github_runner_cluster_docker_cli_environment": {"DOCKER_CONTEXT": f"ctx-{name}"}, + } + for name, role in roles.items() + } + if unreachable: + hosts[unreachable] |= {"ansible_connection": "ssh", "ansible_host": "127.0.0.1", "ansible_port": 1} + return self.run_checks(hosts, "--limit", run_on, *extra, group_vars=common, check_tasks="running_version", timeout=timeout) + + def container(self, name: str, version: str | None = None, nodes: str = "", kubectl_fails: bool = False) -> None: + (self.stub / f"{name}.state").write_text("running" if version else "exited") + if version: + (self.stub / f"{name}.version").write_text(f"k3s version {version} (24fc436e)") + (self.stub / f"{name}.nodes").write_text(nodes) + if kubectl_fails: + (self.stub / f"{name}.kubectl-fails").write_text("") + + def calls(self) -> list[str]: + log = self.stub / "calls.log" + return log.read_text().splitlines() if log.exists() else [] + + def test_a_first_bootstrap_with_nothing_running_passes_and_says_so(self) -> None: + result = self.running(pin="v1.40.0-k3s1") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + for name in ("node1", "node2", "node3"): + self.assertIn(f"ctx-{name} ps --all --filter name=^/{name}-k3s$ --format {{{{.State}}}}", self.calls()) + self.assertIn("runs its container, so there is no cluster to compare with yet", result.stdout) + + def test_a_read_that_answers_says_nothing_about_a_first_bootstrap(self) -> None: + self.container("node1-k3s", "v1.34.1+k3s1", "node1 v1.34.1+k3s1 true\n") + result = self.running() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("no cluster to compare with yet", result.stdout) + self.assertNotIn("only this node's own container was compared", result.stdout) + + def test_a_hung_docker_daemon_on_another_server_is_cut_off_and_named(self) -> None: + self.container("node1-k3s", "v1.34.1+k3s1", "node1 v1.34.1+k3s1 true\n") + (self.stub / "hang-ctx-node3").write_text("60") + result = self.running("-e", "github_runner_cluster_cluster_read_timeout=2", roles={"node2": "agent"}, timeout=40) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + (self.stub / "daemon-down-ctx-node1").write_text("") + result = self.running("-e", "github_runner_cluster_cluster_read_timeout=2", roles={"node2": "agent"}, timeout=40) + self.assertNotEqual(result.returncode, 0) + self.assertIn("node3 did not answer within github_runner_cluster_cluster_read_timeout", result.stdout) + + def test_a_hung_docker_daemon_on_this_host_is_cut_off_at_its_own_read(self) -> None: + (self.stub / "hang-ctx-node2").write_text("60") + result = self.running("-e", "github_runner_cluster_cluster_read_timeout=2", timeout=40) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(failed_task(result.stdout), "Read the k3s version this node's running container reports") + + def test_the_documented_upgrade_passes_servers_first_then_agents(self) -> None: + """README, Versions: the new pin is written to every host by one run of the role, then the servers are rebuilt by hand, then the agents; the role runs on the whole group at each step.""" + roles = {"node1": "server", "node2": "agent", "node3": "server"} + def cluster(server: str, agent: str) -> None: + nodes = f"node1 {server} true\nnode2 {agent}\nnode3 {server} true\n" + for name, version in (("node1-k3s", server), ("node3-k3s", server)): + self.container(name, version, nodes) + self.container("github-runner-k3s-agent", agent) + (self.stub / "github-runner-k3s-agent.image").write_text("") + for server, agent in (("v1.34.1+k3s1", "v1.34.1+k3s1"), ("v1.35.0+k3s1", "v1.34.1+k3s1"), ("v1.35.0+k3s1", "v1.35.0+k3s1")): + with self.subTest(servers=server, agent=agent): + cluster(server, agent) + result = self.running(pin="v1.35.0-k3s1", roles=roles, run_on="all") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("ctx-node2 image ls --quiet github-runner-k3s-agent", self.calls()) + self.assertEqual(sorted(set(re.findall(r"skipping: \[(node\d)\]", result.stdout.split("Refuse a k3s version outside the upgrade path")[-1]))), ["node1", "node2", "node3"]) + + def test_an_agent_built_above_the_servers_is_refused(self) -> None: + roles = {"node1": "server", "node2": "agent", "node3": "server"} + self.container("node1-k3s", "v1.34.1+k3s1", "node1 v1.34.1+k3s1 true\nnode3 v1.34.1+k3s1 true\n") + result = self.running(pin="v1.35.0-k3s1", roles=roles) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Server node1 runs v1.34.1+k3s1, an older minor than v1.35.0-k3s1; an agent's kubelet may not be newer than the API servers", result.stdout) + self.assertIn("ctx-node2 image ls --quiet github-runner-k3s-agent", self.calls()) + + def test_a_read_timeout_that_is_not_a_positive_whole_number_is_refused(self) -> None: + result = self.running("-e", '{"github_runner_cluster_cluster_read_timeout": "30\\n"}') + self.assertNotEqual(result.returncode, 0) + self.assertIn("it must be a positive", result.stdout) + for value in ("0", "30s", "-1"): + with self.subTest(value=value): + result = self.running("-e", f"github_runner_cluster_cluster_read_timeout={value}") + self.assertNotEqual(result.returncode, 0) + self.assertIn("it must be a positive", result.stdout) + + def test_an_unreachable_server_is_named_and_another_server_still_answers(self) -> None: + self.container("node1-k3s", "v1.32.0+k3s1", "node1 v1.32.0+k3s1 true\n") + result = self.running(roles={"node2": "agent"}, unreachable="node3") + self.assertNotEqual(result.returncode, 0) + self.assertIn("Node node1 runs v1.32.0+k3s1, more than one minor away", result.stdout) + (self.stub / "node1-k3s.kubectl-fails").write_text("") + result = self.running(roles={"node2": "agent"}, unreachable="node3") + self.assertNotEqual(result.returncode, 0) + self.assertIn("node3 could not be reached", result.stdout) + + def test_a_server_reads_the_cluster_through_every_server_with_their_contexts(self) -> None: + nodes = "node1 v1.34.1+k3s1\nnode2 v1.34.1+k3s1\nnode3 v1.34.1+k3s1\n" + for name in ("node1", "node2", "node3"): + self.container(f"{name}-k3s", "v1.34.1+k3s1", nodes) + result = self.running(pin="v1.35.0-k3s1") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("ctx-node2 exec node2-k3s k3s --version", self.calls()) + for name in ("node1", "node2", "node3"): + self.assertIn(f"ctx-{name} exec {name}-k3s kubectl get nodes --request-timeout=30s -o jsonpath={{range .items[*]}}{{.metadata.name}}{{\" \"}}{{.status.nodeInfo.kubeletVersion}}{{\" \"}}{{.metadata.labels.node-role\\.kubernetes\\.io/control-plane}}{{\"\\n\"}}{{end}}", self.calls()) + + def test_a_version_two_minors_above_the_cluster_is_refused(self) -> None: + self.container("node2-k3s", "v1.34.1+k3s1", "node1 v1.34.1+k3s1\nnode2 v1.34.1+k3s1\n") + result = self.running(pin="v1.36.0-k3s1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("more than one minor away from v1.36.0-k3s1", result.stdout) + self.assertIn("github_runner_cluster_allow_k3s_version_jump=true", result.stdout) + + def test_both_reads_run_under_check(self) -> None: + for role, own in (("server", "v1.34.1+k3s1"), ("agent", None)): + with self.subTest(role=role): + self.container("node2-k3s", own, "node1 v1.33.0+k3s1\n") + self.container("node1-k3s", "v1.34.1+k3s1", "node1 v1.33.0+k3s1\n") + result = self.running("--check", pin="v1.35.0-k3s1", roles={"node2": role}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Node node1 runs v1.33.0+k3s1", result.stdout) + + def test_the_own_container_read_runs_under_check(self) -> None: + self.container("node2-k3s", "v1.36.0+k3s1") + result = self.running("--check", pin="v1.35.0-k3s1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("This node's own container (node2) runs v1.36.0+k3s1", result.stdout) + + def test_the_default_is_refused_on_a_cluster_that_runs_a_newer_minor(self) -> None: + self.container("node2-k3s", "v1.35.2+k3s1", "node1 v1.35.2+k3s1\nnode2 v1.35.2+k3s1\n") + result = self.running() + self.assertNotEqual(result.returncode, 0) + self.assertIn(f"newer than {TESTED}; k3s cannot be downgraded", result.stdout) + + def test_a_stopped_bootstrap_server_does_not_hide_a_newer_cluster(self) -> None: + self.container("node1-k3s") + for name in ("node2", "node3"): + self.container(f"{name}-k3s", "v1.36.0+k3s1", "node1 v1.34.1+k3s1 true\nnode2 v1.36.0+k3s1 true\nnode3 v1.36.0+k3s1 true\n") + result = self.running(run_on="node1") + self.assertNotEqual(result.returncode, 0) + self.assertIn(f"Server node2 runs v1.36.0+k3s1, a newer minor than {TESTED}", result.stdout) + + def test_a_bootstrap_server_whose_api_fails_does_not_stop_a_read_through_another(self) -> None: + self.container("node1-k3s", "v1.34.1+k3s1", kubectl_fails=True) + self.container("node3-k3s", "v1.34.1+k3s1", "node1 v1.34.1+k3s1\nnode3 v1.34.1+k3s1\n") + result = self.running(roles={"node2": "agent"}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_a_cluster_no_server_can_be_read_through_is_refused_and_the_override_skips_the_reads(self) -> None: + self.container("node1-k3s", "v1.34.1+k3s1", kubectl_fails=True) + (self.stub / "daemon-down-ctx-node3").write_text("") + result = self.running(roles={"node2": "agent"}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("The cluster could not be read through any server of the inventory", result.stdout) + self.assertIn("node1 could not be read (The connection to the server 127.0.0.1:6443 was refused)", result.stdout) + self.assertIn("node3 could not be read (Cannot connect to the Docker daemon)", result.stdout) + (self.stub / "calls.log").unlink() + result = self.running("-e", "github_runner_cluster_allow_k3s_version_jump=true", roles={"node2": "agent"}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.calls(), []) + + def test_the_override_lets_one_run_past_the_refusal(self) -> None: + self.container("node2-k3s", "v1.35.2+k3s1", "node2 v1.35.2+k3s1\n") + result = self.running("-e", "github_runner_cluster_allow_k3s_version_jump=true") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_an_agent_reads_the_cluster_through_the_servers_with_their_contexts(self) -> None: + self.container("node1-k3s", "v1.34.1+k3s1", "node1 v1.34.1+k3s1\nnode2 v1.33.0+k3s1\n") + result = self.running(pin="v1.35.0-k3s1", roles={"node2": "agent"}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Node node2 runs v1.33.0+k3s1", result.stdout) + self.assertIn("ctx-node2 ps --all --filter name=^/github-runner-k3s-agent$ --format {{.State}}", self.calls()) + self.assertFalse([call for call in self.calls() if "node2-k3s" in call], "an agent read a server container on its own host") + + def test_a_group_with_no_bootstrap_host_reads_its_servers(self) -> None: + self.container("node3-k3s", "v1.32.0+k3s1", "node3 v1.32.0+k3s1 true\n") + result = self.running(roles={"node2": "agent"}, bootstrap=None) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Node node3 runs v1.32.0+k3s1, more than one minor away", result.stdout) + + def test_a_running_server_that_cannot_list_its_nodes_alone_refuses(self) -> None: + self.container("node2-k3s", "v1.34.1+k3s1", kubectl_fails=True) + result = self.running(pin="v1.34.1-k3s1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("node2 could not be read", result.stdout) + + def test_a_docker_daemon_that_cannot_be_reached_fails_the_run(self) -> None: + (self.stub / "daemon-down").write_text("") + result = self.running(pin="v1.34.1-k3s1") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(failed_task(result.stdout), "Read the k3s version this node's running container reports") + + def bootstrap_shaped(self, role: str) -> subprocess.CompletedProcess[str]: + """The topology and version checks on the one-host inventory bootstrap.sh builds: group all, joining a server outside the inventory.""" + inventory = self.work / "bootstrap.ini" + inventory.write_text("localhost ansible_connection=local\n") + variables = self.work / "bootstrap.json" + variables.write_text(json.dumps({ + "github_runner_cluster_group": "all", + "github_runner_cluster_node_role": role, + "github_runner_cluster_node_name": "joiner", + "github_runner_cluster_server_url": "https://server.example.internal:6443", + "github_runner_cluster_tls_sans": [], + "github_runner_cluster_docker_cli_environment": {}, + "check_tasks": "topology,running_version", + "check_hosts": "all", + })) + args = [ANSIBLE_PLAYBOOK, "-i", str(inventory), str(ROOT / "tests" / "cluster_versions" / "checks.yml"), "-e", f"@{variables}"] + env = ansible_env() | {"PATH": f"{self.work / 'bin'}{os.pathsep}{os.environ['PATH']}", "DOCKER_STUB": str(self.stub)} + return subprocess.run(args, capture_output=True, text=True, check=False, env=env, cwd=self.work) + + def test_a_bootstrap_sh_agent_compares_only_its_own_container_and_says_why(self) -> None: + self.container("github-runner-k3s-agent", "v1.36.0+k3s1") + result = self.bootstrap_shaped("agent") + self.assertNotEqual(result.returncode, 0) + self.assertIn("This node's own container (joiner) runs v1.36.0+k3s1", result.stdout) + self.assertIn("No server of all is in the inventory", result.stdout) + (self.stub / "github-runner-k3s-agent.state").write_text("exited") + result = self.bootstrap_shaped("agent") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_a_bootstrap_sh_joining_server_reads_its_own_container(self) -> None: + result = self.bootstrap_shaped("server") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("all joins a cluster whose other servers are outside the inventory, and none of its own servers runs its container", result.stdout) + self.assertNotIn("no cluster to compare with yet", result.stdout) + self.container("github-runner-k3s", "v1.34.1+k3s1", "joiner v1.34.1+k3s1 true\nother v1.36.0+k3s1 true\n") + result = self.bootstrap_shaped("server") + self.assertNotEqual(result.returncode, 0) + self.assertIn("Server other runs v1.36.0+k3s1, a newer minor than", result.stdout) + + def test_a_compose_file_path_with_a_colon_is_refused(self) -> None: + for extra in ('{"github_runner_cluster_compose_files": ["extra:1.yml"]}', '{"github_runner_cluster_dir": "/srv/node:1"}'): + with self.subTest(extra=extra): + result = self.run_checks({"node1": {"github_runner_cluster_dir": "/srv/node1"}}, "-e", extra, check_tasks="compose_files") + self.assertNotEqual(result.returncode, 0) + self.assertIn("cannot hold a path with a colon in it", result.stdout) + result = self.run_checks({"node1": {"github_runner_cluster_dir": "/srv/node1"}}, "-e", '{"github_runner_cluster_compose_files": ["/srv/extra.yml"]}', check_tasks="compose_files") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def render_env(self, *extra: str) -> tuple[list[str], list[str]]: + env, agent = self.work / "env", self.work / "env-agent" + args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", f"env_agent_output={agent}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra] + result = subprocess.run(args, capture_output=True, text=True, check=False, env=ansible_env(), cwd=self.work) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return env.read_text().splitlines(), agent.read_text().splitlines() + + def test_the_env_files_carry_both_versions(self) -> None: + for lines in self.render_env("-e", "github_runner_cluster_tailscale_version=v1.90.0"): + self.assertIn(f"K3S_VERSION={TESTED}", lines) + self.assertIn("TAILSCALE_VERSION=v1.90.0", lines) + + def test_the_env_files_name_the_compose_files_and_project(self) -> None: + for lines in self.render_env("-e", '{"github_runner_cluster_compose_files": ["compose.extra.yml", "/elsewhere/absolute.yml"], "github_runner_cluster_compose_project": "example"}'): + self.assertIn("COMPOSE_FILE=/srv/github-runner/docker-compose.yml:/srv/github-runner/compose.extra.yml:/elsewhere/absolute.yml", lines) + self.assertIn("COMPOSE_PROJECT_NAME=example", lines) + + @unittest.skipUnless(has_compose(), "docker compose is not installed") + def test_a_compose_command_run_by_hand_uses_the_role_s_files_and_project(self) -> None: + project = self.work / "project" + project.mkdir() + server_env, _ = self.render_env("-e", json.dumps({"github_runner_cluster_compose_files": ["compose.extra.yml"], "github_runner_cluster_compose_project": "example", "github_runner_cluster_dir": str(project)})) + shutil.copy(COMPOSE, project / "docker-compose.yml") + (project / "compose.extra.yml").write_text("services:\n k3s:\n labels:\n example.extra: applied\n") + (project / ".env").write_text("\n".join(server_env) + "\n") + clean = {key: value for key, value in os.environ.items() if not key.startswith(("DOCKER_", "COMPOSE_", "K3S_", "TAILSCALE_"))} + for cwd, args in ((project, []), (self.work, ["--project-directory", str(project)])): + with self.subTest(cwd=str(cwd)): + result = subprocess.run(["docker", "compose", *args, "--profile", "server", "config", "--format", "json"], capture_output=True, text=True, check=False, cwd=cwd, env=clean) + self.assertEqual(result.returncode, 0, result.stderr) + config = json.loads(result.stdout) + self.assertEqual(config["name"], "example") + self.assertEqual(config["services"]["k3s"]["labels"], {"example.extra": "applied"}) + + +class BootstrapTest(unittest.TestCase): + """bootstrap.sh run against stand-ins for its virtualenv's ansible-galaxy and ansible-playbook, which record the variables file it passes.""" + + def setUp(self) -> None: + self.work = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.work) + shutil.copy(ROOT / "bootstrap.sh", self.work / "bootstrap.sh") + (self.work / "ansible").mkdir() + venv = self.work / ".ansible-venv" / "bin" + venv.mkdir(parents=True) + for name, body in (("ansible-galaxy", "exit 0"), ("ansible-playbook", 'for arg in "$@"; do case "$arg" in @*) cp "${arg#@}" "$CAPTURE";; esac; done')): + (venv / name).write_text(f"#!/bin/sh\n{body}\n") + (venv / name).chmod(0o755) + + def variables(self, **settings: str) -> dict[str, Any]: + (self.work / ".env.bootstrap").write_text("".join(f"{key}={value}\n" for key, value in settings.items())) + capture = self.work / "vars.json" + result = subprocess.run(["bash", str(self.work / "bootstrap.sh")], capture_output=True, text=True, check=False, env={**os.environ, "CAPTURE": str(capture)}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + loaded: dict[str, Any] = json.loads(capture.read_text()) + return loaded + + @unittest.skipIf(shutil.which("jq") is None, "jq is not installed") + def test_empty_versions_leave_the_role_s_pinned_defaults(self) -> None: + variables = self.variables(K3S_TOKEN="t", K3S_VERSION="", TAILSCALE_VERSION="") + self.assertNotIn("github_runner_cluster_k3s_version", variables) + self.assertNotIn("github_runner_cluster_tailscale_version", variables) + + @unittest.skipIf(shutil.which("jq") is None, "jq is not installed") + def test_set_versions_are_passed_through(self) -> None: + variables = self.variables(K3S_TOKEN="t", K3S_VERSION="v1.35.0-k3s1", TAILSCALE_VERSION="v1.90.0") + self.assertEqual(variables["github_runner_cluster_k3s_version"], "v1.35.0-k3s1") + self.assertEqual(variables["github_runner_cluster_tailscale_version"], "v1.90.0") + + +if __name__ == "__main__": + unittest.main()