diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000..5ced3e2 --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,5 @@ +--- +# Paths that are not Ansible content and that ansible-lint's YAML loader cannot read. +exclude_paths: + # A Compose file, which empties the node service's ports with Compose's !reset tag. + - roles/github_runner_cluster/files/docker-compose.host-network.yml diff --git a/.github/workflows/mesh-integration.yml b/.github/workflows/mesh-integration.yml index 824bbe0..e357f07 100644 --- a/.github/workflows/mesh-integration.yml +++ b/.github/workflows/mesh-integration.yml @@ -41,6 +41,7 @@ jobs: - {scenario: compose_faulty, compose: v2.38.2} - {scenario: in_cluster, compose: v5.5.1, pin: true} - {scenario: context_checks, compose: v5.5.1} + - {scenario: host_network, compose: v5.5.1} steps: - uses: actions/checkout@v7 diff --git a/README.md b/README.md index 76613d5..6b77510 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ ARC's `gha-runner-scale-set` solves both by design: it runs one fresh pod per jo The control plane itself runs genuine embedded-etcd HA across three `server`-role hosts, not a single node with worker capacity bolted on: one host bootstraps with `--cluster-init` (the one supported migration path from k3s's default single-node SQLite datastore to embedded etcd), and two further hosts join as full `k3s server` members - a real vote each in etcd's own raft quorum, not just schedulable capacity. 3 is the practical minimum: etcd tolerates the loss of any *minority* of its members without losing quorum, and with 2 servers that minority is zero - a second server can only ever make quorum strictly worse than staying at one, never better, so "2-node HA" isn't a real intermediate step, it's a trap. Getting there over Tailscale rather than a shared LAN needed real, non-obvious work - see the [Non-obvious behaviour](#non-obvious-behaviour) section's etcd-specific entries for what actually broke and why. -Two or more physical hosts on different home networks can't reach each other's LAN IPs directly, so cross-node cluster traffic (a second node joining, cross-node pod-to-pod traffic, the API server's proxy to a remote kubelet, and now etcd's own peer traffic between `server` hosts) needs a real overlay between them - see [Tailscale](#tailscale) below for how that overlay is built and the DNS pitfall it introduces. +Hosts on different networks can't reach each other's LAN addresses, and even hosts that share a LAN can't from inside their Docker VMs by default: each node's k3s container sits behind its VM's NAT, so its own Tailscale client has no address to offer its peers and every pair of nodes is relayed. So cross-node cluster traffic (a node joining, cross-node pod-to-pod traffic, the API server's proxy to a remote kubelet, and etcd's own peer traffic between `server` hosts) runs over a real overlay with mesh addresses - see [Tailscale](#tailscale) below for how that overlay is built and the DNS pitfall it introduces. Where hosts do share a LAN, `github_runner_cluster_network_mode: host`, on a Docker VM that has an address of its own on that LAN, lets their Tailscale clients reach each other directly over it, with the relay as the fallback when a host leaves; nothing k3s uses changes (see `roles/github_runner_cluster/README.md`, Networking). The custom runner toolchain (GCC 11/G++ for C++20, Bun, the `gh` CLI — see `Dockerfile`) carries over from the previous design, rebased onto ARC's own [`actions/actions-runner`](https://github.com/actions/runner/pkgs/container/actions-runner) base image instead of `catthehacker/ubuntu:act-latest`. ARC's runner pods use their own registration flow, so the previous design's vendored [myoung34/docker-github-actions-runner](https://github.com/myoung34/docker-github-actions-runner) scripts (registration, GitHub App JWT signing, ephemeral re-registration) are not needed. diff --git a/plugins/filter/cluster_guard.py b/plugins/filter/cluster_guard.py new file mode 100644 index 0000000..7cb2218 --- /dev/null +++ b/plugins/filter/cluster_guard.py @@ -0,0 +1,178 @@ +"""Filters the github_runner_cluster role uses before it moves a node container between networks: whether recreating it would be safe for the cluster, and whether the Docker host has an address on the LAN.""" + +from __future__ import annotations + +import ipaddress +import json +from collections.abc import Mapping, Sequence +from typing import Any + +from ansible.errors import AnsibleFilterError + +# The label ARC puts on every ephemeral runner pod (actions-runner-controller, controllers/actions.github.com/resourcebuilder.go, newEphemeralRunnerPod). +RUNNER_POD_LABEL = "actions-ephemeral-runner" +RUNNER_POD_LABEL_VALUE = "True" +# Labels k3s gives a server node (one that runs the API server and, on etcd, an etcd member). +SERVER_NODE_LABELS = ("node-role.kubernetes.io/control-plane", "node-role.kubernetes.io/etcd", "node-role.kubernetes.io/master") +FINISHED_POD_PHASES = ("Succeeded", "Failed") + + +def _labels(item: Mapping[str, Any]) -> Mapping[str, Any]: + labels = item.get("metadata", {}).get("labels") or {} + return labels if isinstance(labels, Mapping) else {} + + +def _ready(node: Mapping[str, Any]) -> bool: + conditions = node.get("status", {}).get("conditions") or [] + return any(condition.get("type") == "Ready" and condition.get("status") == "True" for condition in conditions) + + +def _is_server(node: Mapping[str, Any]) -> bool: + labels = _labels(node) + return any(str(labels.get(label, "")).lower() == "true" for label in SERVER_NODE_LABELS) + + +def _etcd_healthy(result: Any) -> bool: + """Whether one server's etcd read (an Ansible loop result of ``kubectl get --raw /readyz/etcd`` in its container) answered "ok".""" + if not isinstance(result, Mapping) or result.get("unreachable") or result.get("skipped"): + return False + return result.get("rc") == 0 and str(result.get("stdout", "")).strip() == "ok" + + +def cluster_restart_blockers(nodes: Any, pods: Any, node_name: Any, servers: Any, etcd_results: Any) -> list[str]: + """Say why restarting (or recreating) this host's node now would be unsafe, or return an empty list when it would not be. + + A server node may restart only while the other servers whose node is Ready and whose own API server reports etcd healthy still form a majority of every server, so etcd keeps its quorum without this one; a cluster with this node as its only server has no quorum to lose. Whatever cannot be told counts against the restart: a server the inventory names whose node is missing or not labelled as a server, a node labelled as a server that the inventory does not name (its etcd was not read), and a server whose etcd read failed or could not reach its host. Any node may restart only while no ARC runner pod that has not finished is on it, since restarting the node ends the job that pod runs. + + :param nodes: the cluster's Node objects (``kubectl get nodes -o json``'s ``items``). + :param pods: Pod objects (``kubectl get pods -A -o json``'s ``items``); pods on other nodes are ignored. + :param node_name: this host's node name. + :param servers: the node names of the cluster's servers as the inventory decides them (tasks/topology.yml), this host's included when it is one. + :param etcd_results: the loop results of reading ``/readyz/etcd`` through each other server's container, each with the ``item`` it looped over: that server's entry of the topology's hosts as ``dict2items`` gives it (``key`` the host, ``value.node_name`` its node name); empty when this host is not a server. + :returns: one sentence per reason not to restart, or an empty list. + """ + if not all(isinstance(value, Sequence) and not isinstance(value, str) for value in (nodes, pods, servers, etcd_results)): + raise AnsibleFilterError("The nodes, pods, servers and etcd results must be lists.") + blockers: list[str] = [] + by_name = {node.get("metadata", {}).get("name"): node for node in nodes} + if node_name not in by_name: + blockers.append(f"This host's node, {node_name}, is not in the cluster's node list, so whether restarting it is safe cannot be told.") + return blockers + known = [str(name) for name in servers] + unlabelled = sorted(name for name in known if name in by_name and not _is_server(by_name[name])) + if unlabelled: + blockers.append(f"The cluster does not label {', '.join(unlabelled)} as server nodes, though the inventory makes them servers, so whether etcd keeps its quorum cannot be told.") + unknown = sorted(name for name, node in by_name.items() if _is_server(node) and name not in known) + every_server = sorted(set(known) | set(unknown)) + if node_name in known and len(every_server) > 1: + etcd_ok = { + str(result["item"].get("value", {}).get("node_name")): _etcd_healthy(result) + for result in etcd_results + if isinstance(result, Mapping) and isinstance(result.get("item"), Mapping) + } + healthy = sorted(name for name in every_server if name != node_name and name in by_name and _ready(by_name[name]) and etcd_ok.get(name, False)) + majority = len(every_server) // 2 + 1 + if len(healthy) < majority: + unhealthy = [] + for name in every_server: + if name == node_name or name in healthy: + continue + if name not in by_name: + unhealthy.append(f"{name} (not in the node list)") + elif not _ready(by_name[name]): + unhealthy.append(f"{name} (not Ready)") + elif name in unknown: + unhealthy.append(f"{name} (not in the inventory, so its etcd was not read)") + else: + unhealthy.append(f"{name} (etcd not reported healthy)") + blockers.append( + f"Restarting {node_name} would leave {len(healthy)} of {len(every_server)} servers with a Ready node and a healthy etcd," + f" fewer than the {majority} etcd needs for quorum: {', '.join(unhealthy)}." + ) + runners = sorted( + pod.get("metadata", {}).get("namespace", "") + "/" + pod.get("metadata", {}).get("name", "") + for pod in pods + if pod.get("spec", {}).get("nodeName") == node_name + and str(_labels(pod).get(RUNNER_POD_LABEL, "")) == RUNNER_POD_LABEL_VALUE + and pod.get("status", {}).get("phase") not in FINISHED_POD_PHASES + ) + if runners: + blockers.append(f"Runner pods are still on {node_name} and would lose their jobs: {', '.join(runners)}. Cordon the node and wait for them to finish.") + return blockers + + +# How ansible-core reports a task cut off by its timeout keyword: from 2.18 the result has a "timedout" key; before that only its message says so (as in plugins/filter/cluster_versions.py). +TIMEOUT_MESSAGES = ("failed to execute in the expected time frame", "Timed out after") + + +def cluster_guard_state(results: Any) -> dict[str, Any]: + """Take the cluster's state from the first server that answered the guard's read (tasks/host_network.yml), or say why none did. + + :param results: the loop results of reading the nodes and this node's pods through each server of the inventory, each with its host as ``item`` and, on success, ``stdout`` holding ``{"nodes": ..., "pods": ...}``. + :returns: ``{"state": {"nodes": [...], "pods": [...]}, "problem": ""}`` from the first that answered, or ``{"state": None, "problem": ...}`` saying why for each server, or that the inventory holds none. + """ + if not isinstance(results, Sequence) or isinstance(results, str): + raise AnsibleFilterError("The reads must be a list of loop results.") + if not results: + return { + "state": None, + "problem": "The inventory holds no server of this cluster (its servers are outside it, as for a host bootstrap.sh joins to a cluster), so the role cannot read the cluster to tell whether moving this node is safe. Move it by hand, following the role README's Networking section.", + } + reasons: list[str] = [] + for result in results: + if not isinstance(result, Mapping): + raise AnsibleFilterError(f"{result!r} is not a loop result.") + host = str(result.get("item", "?")) + if result.get("unreachable"): + reasons.append(f"{host} could not be reached") + continue + message = str(result.get("msg", "") or "") + if result.get("timedout") or any(text in message for text in TIMEOUT_MESSAGES): + reasons.append(f"{host} did not answer within github_runner_cluster_cluster_read_timeout") + continue + if result.get("rc") != 0: + lines = [line.strip() for line in str(result.get("stderr", "") or message).splitlines() if line.strip()] + reasons.append(f"{host} could not be read ({lines[-1] if lines else 'exit status ' + str(result.get('rc'))})") + continue + try: + state = json.loads(str(result.get("stdout", ""))) + except ValueError: + reasons.append(f"{host} gave an answer that is not the cluster's state") + continue + if isinstance(state, Mapping) and isinstance(state.get("nodes"), Mapping) and isinstance(state.get("pods"), Mapping): + return {"state": {"nodes": state["nodes"].get("items", []), "pods": state["pods"].get("items", [])}, "problem": ""} + reasons.append(f"{host} gave an answer that is not the cluster's state") + return {"state": None, "problem": f"The cluster could not be read through any server of the inventory, so whether moving this node is safe cannot be told: {'; '.join(reasons)}."} + + +def addresses_in_network(lines: Any, network: Any) -> list[str]: + """Return the IPv4 addresses in ``ip -4 -o addr show`` output that lie in a network. + + :param lines: the command's output lines, one address per line, such as ``2: eth0 inet 192.0.2.10/24 brd ...``. + :param network: the network in CIDR notation, for example ``192.0.2.0/24``. + :returns: the matching addresses, without their prefix length, in the order listed. + :raises AnsibleFilterError: if the network is not a valid IPv4 network or the lines are not a list. + """ + try: + wanted = ipaddress.IPv4Network(str(network).strip(), strict=False) + except ValueError as error: + raise AnsibleFilterError(f"{network!r} is not an IPv4 network in CIDR notation, such as 192.0.2.0/24: {error}") from error + if not isinstance(lines, Sequence) or isinstance(lines, str): + raise AnsibleFilterError("The address lines must be a list.") + found: list[str] = [] + for line in lines: + fields = str(line).split() + if "inet" not in fields or fields.index("inet") + 1 >= len(fields): + continue + address = ipaddress.IPv4Interface(fields[fields.index("inet") + 1]).ip + if address in wanted: + found.append(str(address)) + return found + + +class FilterModule: + """Registers the cluster guard filters with Ansible.""" + + def filters(self) -> dict[str, Any]: + """Return the filters this plugin provides.""" + return {"cluster_restart_blockers": cluster_restart_blockers, "cluster_guard_state": cluster_guard_state, "addresses_in_network": addresses_in_network} diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index f5ac7f4..a6c15d1 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -119,6 +119,69 @@ The limitation: the control server lives on one node. While the bootstrap server Headscale's database lives on the bootstrap server's k3s data volume, and a static pod cannot use a replicated persistent volume instead, since kubelet runs it with no API server to bind a claim. So the provider requires Litestream: set `github_runner_cluster_headscale_litestream_replica` (see [Backups and a warm standby](#backups-and-a-warm-standby) for the settings), and the pod gets a `litestream-restore` init container and a `litestream` sidecar. A rebuilt bootstrap server then restores the database before Headscale starts, with the same data loss as `headscale_hosted`. The Noise key copy in `keys/` sits on the host, outside the k3s volume. It survives losing that volume but not the host, so supply `github_runner_cluster_headscale_noise_private_key` too if the host itself may be replaced. To run without Litestream, set `github_runner_cluster_headscale_accept_node_local_state: true`, and accept that losing that volume loses every registration: every node must then join again with a new key. There is no standby for this provider; recovery is rebuilding the bootstrap server. +## Networking + +Every node's k3s uses its mesh address for everything: its node IP, etcd's peer URLs, the API server and pod traffic. Which physical path the mesh takes between two nodes is Tailscale's choice, among the endpoints each node's Tailscale client can offer its peers. On Compose's default bridge network (`github_runner_cluster_network_mode: bridge`, the default) the client sits behind the Docker host's NAT and, inside a Docker VM, behind the VM's too, so it has no address its peers can reach and every pair of nodes is relayed, even on one LAN. + +`github_runner_cluster_network_mode: host` runs the node's container on the Docker host's own network instead (`docker-compose.host-network.yml`, merged last). Its client then offers the host's own addresses, so nodes that share a network connect directly and fall back to a relay or a hole-punched path when they do not. The container publishes no port, since its ports are the host's; with `github_runner_cluster_wait_for_nodes` or `github_runner_cluster_record_deployed_version` on, the role checks that the API server answers on the host's loopback before it uses the kubeconfig there. k3s's ports, routes and firewall rules, Tailscale's, and the entrypoint's unreachable route for the mesh ranges then live in the host's namespace, shared with any other container on the host network there. Nothing k3s uses changes, so a node moves between the two modes with a container recreate. + +On the host network the node's ports are the Docker host's. The API server (6443) and kubelet (10250) then answer on every address the host has, so on a bridged VM they are reachable from the LAN on the VM's address; etcd stays on loopback and the mesh address. In bridge mode only the API server's port is published, on the Mac's own addresses. Other containers on the same Docker host also gain a route into the mesh through the node's Tailscale routing, so give a node on the host network a VM of its own. + +On a Linux machine running Docker without a VM, the host network is the machine's own. The node's Tailscale client would then create `tailscale0`, Tailscale's routing table and rules and its firewall chains beside the machine's own Tailscale client, and a second node on the same Docker host would share all of that and the kubelet's port. Neither works, so the role refuses host mode when the Docker host's network already has a Tailscale device that is not the node's own (as it is once the node runs on the host network), and a node that should go direct on such a machine needs a VM of its own. + +Host mode needs the Docker host to have an address on the LAN the nodes share, given as `github_runner_cluster_lan_cidr`: before starting the node, the role reads the host's addresses from a throwaway container on the host network (`github_runner_cluster_network_probe_image`) and fails, changing nothing, when none lies in that network. Whenever the node's running container is on the other kind of network than the one asked for, Compose is about to recreate it, so the role first refuses unless the cluster stays safe without the node: the other servers whose node is Ready and whose own API server reports etcd healthy (`/readyz/etcd`, read through each server's container on its host) must still be a majority of all servers, and no ARC runner pod that has not finished may be on the node (cordon it and let its jobs finish). What cannot be told refuses the move: a cluster that cannot be read, servers the cluster does not label as servers, a server node that is missing, a server the inventory does not hold, or a server that cannot be reached. Only one node may move per run, since each host's checks see the others as healthy before any is recreated. All of this runs before the role changes anything, under `--check` too, and only reads (the address probe pulls its small image if the host lacks it). The check does not see which server leads etcd; the runbook below moves leadership off a node by hand. + +On macOS the Docker host is a VM, which needs an address of its own on the LAN, and the role does not create or change VMs. With Colima that means a profile created bridged: a profile's network mode is fixed when it is created and its LAN address cannot be turned off afterwards, so an existing user-mode VM cannot be converted, and a node moves to a new profile instead. Docker Desktop has no bridged VM network. + +### Moving a node to a bridged Colima VM + +One node at a time. The node keeps its identity because its two volumes move with it: the k3s data (etcd member, certificates, token) and the Tailscale state, so its mesh address, node IP and etcd peer URL stay the same and nothing re-joins. Run every `docker` command below against the engine it names (`docker --context ...`). + +Before stopping anything: + +1. **Check etcd on every server.** On each server, `docker exec kubectl get --raw '/readyz?verbose'` lists `[+]etcd ok`. k3s ships no `etcdctl`, so read the members' health and the leader with one in a throwaway container that shares a healthy server's network namespace and reads its certificates from the k3s data volume, using an etcd image of the version the k3s release embeds (its release notes name it): + + ``` + etcdctl() { + docker run --rm --network container: -v _k3s-data:/var/lib/rancher/k3s:ro \ + --entrypoint etcdctl gcr.io/etcd-development/etcd: \ + --cacert /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \ + --cert /var/lib/rancher/k3s/server/tls/etcd/client.crt \ + --key /var/lib/rancher/k3s/server/tls/etcd/client.key "$@" + } + etcdctl --endpoints https://127.0.0.1:2379 endpoint health --cluster + etcdctl --endpoints https://127.0.0.1:2379 endpoint status --cluster -w table + ``` + + Every member must be healthy and started, and the other servers must stay a majority without this one. Record each member's ID and peer URL for the checks after the move. +2. **Move etcd leadership off the node** if `endpoint status` shows it as the leader. k3s has no command for it ([k3s docs, "Using etcdctl"](https://docs.k3s.io/advanced), [archived](http://web.archive.org/web/2026/https://docs.k3s.io/advanced)); with the function above, run `etcdctl --endpoints https://:2379 move-leader ` and read `endpoint status` again. +3. **Take an etcd snapshot as the last resort,** on another server: `docker exec k3s etcd-snapshot save --name before-network-move`. +4. **Empty the node of jobs.** `kubectl cordon `, then wait until `kubectl get pods --all-namespaces --field-selector spec.nodeName=` lists no runner pod. + +The new engine: + +5. **Choose the bridged interface.** Pick a wired one where the host has one: the VM then reaches its peers without the host's Wi-Fi. A profile bridged to an adapter that is missing when the VM starts (an undocked laptop's dock adapter) does not start at all: `colima start` fails, and every container in that VM stays down until the adapter is back. Bridging over Wi-Fi works, through macOS's MAC NAT, so the router sees a second lease on the Mac's own Wi-Fi MAC. The interface is not fixed when the profile is created: a stopped profile started again with another `--network-interface` came back on that interface in testing, so a node bridged over Wi-Fi now should later move to a wired adapter with a stop and a start of its profile, not another profile and volume move. That has been seen with a test profile, not yet with a node on it. +6. **Give the new profile a home of its own whenever another profile already runs in the default home** (`~/.colima`): VMs in one home share its user-mode network daemon. Create the `COLIMA_HOME` directory before running any `colima` command with it, since Colima falls back silently to `~/.colima` when it does not exist. +7. **Approve Colima's network helper once.** The first start of a profile with an address installs `socket_vmnet` and a sudoers entry for it, which an administrator approves once per host. +8. **Stop the node and keep its image and node password.** Run `docker update --restart=no` and `docker stop` on the node's container, so nothing starts the old copy again. Then `docker save` the node's image under the name its container uses, so the new engine runs the same build rather than building one, and `docker cp :/etc/rancher/node/password` to a file (never print it): the node password lives in the container's own layer, and carrying it avoids a `NodePasswordValidationFailed` event when the node registers again. +9. **Export both volumes with the container stopped,** so the etcd data is consistent. Use a throwaway container that tars each volume to a file. The image's `agent/containerd` directory can be left out of the k3s data; kubelet pulls its images again. +10. **Create the bridged profile**, sized for what the node runs: Colima's default VM is far smaller than a server that also runs runner pods needs. For example `colima start --profile --vm-type vz --cpus --memory --disk --network-address --network-mode bridged --network-interface --activate=false`. `--activate=false` leaves the host's current Docker context alone; without it Colima switches every other Docker client on the host to the new VM. Check that `colima status --profile --json` reports an `ip_address` on the LAN. On a memory-tight host, quit the old Docker engine first. +11. **Import onto the new engine.** `docker load` the image, create the two volumes as Compose would, with its labels (`docker volume create --label com.docker.compose.project= --label com.docker.compose.volume= _`, for `k3s-data` and `k3s-tailscale-data`), and untar each export into its volume. + +Move the engine first and the network after, one change at a time, so that the second change goes through the role's guard: + +12. **Create the node on the new engine, still on the bridge network,** and give it its password: in `github_runner_cluster_dir`, whose `.env` the last run of the role wrote for the bridge network, run `docker --context compose --profile server create --no-build k3s` and `docker --context cp :/etc/rancher/node/password`. +13. **Start it through the role.** Point the host's `github_runner_cluster_docker_context` at the new profile's context, leave `github_runner_cluster_network_mode` as it is, and run the role for this host alone (`--limit `, after a `--check --diff` run). It starts the node from the loaded image, building nothing. This is the first start on the new engine, after which the old copy must never run again (see the rollback limits). +14. **Check the node kept its identity.** It is Ready with the same InternalIP; `endpoint status` from step 1 lists the same member IDs and peer URLs, every member healthy; `tailscale status --self` in the node's container shows the same device and mesh address, and no new device has joined the tailnet; the node's log has no `membershipError`; no `NodePasswordValidationFailed` event names it (`kubectl get events --all-namespaces --field-selector involvedObject.name=`); and `kubeconfig/kubeconfig.yaml` in `github_runner_cluster_dir` has been rewritten since the start, which shows the bind mounts reach the host through the new VM's mount of the host's directories. +15. **Move it to the host network through the role.** Check etcd again and move leadership off the node if it leads (steps 1 and 2), set `github_runner_cluster_network_mode: host` and `github_runner_cluster_lan_cidr`, and run the role for this host alone again. The node's container is now running on the bridge network, so the role's guard reads the cluster before Compose recreates it on the host network, and refuses while the cluster needs the node. Do not start or recreate it by hand: until the role has rewritten `.env`, `COMPOSE_FILE` names only the bridge network's files. +16. **Check it again,** as in step 14, and that `tailscale status` shows `direct` paths to every peer that is bridged too. A node on a user-mode VM never gets a direct path to a bridged one, because macOS Local Network privacy drops the user-mode VM's traffic to other Macs, so the benefit arrives once two nodes are bridged. +17. **Uncordon the node.** +18. **Make the new profile start with the host, and the old one not.** `brew services start colima` runs `colima start -f` for the default profile in the default home only ([Colima FAQ](https://github.com/abiosoft/colima/blob/v0.10.3/docs/FAQ.md), [archived](http://web.archive.org/web/2026/https://github.com/abiosoft/colima/blob/v0.10.3/docs/FAQ.md), and Homebrew's service definition for the formula). A profile with another name or home needs a launchd agent of its own that runs `colima start --profile -f` with its `COLIMA_HOME`, and whatever started the old engine must no longer start the old copy of the node. The collection manages neither. + +**Rollback limits.** Starting the old container again on the old engine is safe only until the node's first start on the new one. After that the moved etcd member has appended to its log and voted, and the old copy would come back behind what the cluster believes it stored. Roll back by exporting the new volumes back the same way, or by removing the node (`kubectl delete node`, which removes its etcd member) and joining it fresh. Never run both copies at once: they are the same mesh node and the same etcd member. + +Tailscale settings a node needs go on the `tailscaled` command line in `k3s/node-entrypoint.sh` or through `--vpn-auth`'s `extraArgs`: k3s runs `tailscale up --reset` on every start, which drops any other preference. + ## Multiple sites Each site is its own cluster: its own inventory group, bootstrap server and mesh provider, run by its own play with `github_runner_cluster_group` set to that group (see `examples/two-site/`). One cluster spanning two meshes is not supported: flannel's tailscale backend carries pod traffic over the single mesh every node belongs to, so every node of a cluster must be on the same tailnet or Headscale server. A host belongs to at most one cluster group. @@ -138,6 +201,7 @@ Each site is its own cluster: its own inventory group, bootstrap server and mesh - `github_runner_cluster_node_taints`: Kubernetes taints (`key=value:Effect`, no spaces) the host's node registers with, empty by default. `NoSchedule` keeps pods without a matching toleration off it and `NoExecute` also evicts the ones already there. k3s applies them on every registration, so they come back if the node object is deleted, which a taint set with `kubectl` does not. - Per-host overrides: `github_runner_cluster_node_role` (`server` or `agent`), `github_runner_cluster_bootstrap` (true picks the bootstrap host, false excludes a host), `github_runner_cluster_server_url` (join this server instead, for a cluster whose servers are outside the inventory), `github_runner_cluster_tls_sans`, `github_runner_cluster_node_address`. The run fails if the result has an even number of servers, no servers, or more than one bootstrap host. - `github_runner_cluster_docker_context` (default empty): pins the Docker context this host's node runs in (see [Docker contexts](#docker-contexts)). +- `github_runner_cluster_network_mode` (default `bridge`, or `host`): the network the node's container runs on, and with `host`, `github_runner_cluster_lan_cidr` (required: the LAN the nodes share) and `github_runner_cluster_network_probe_image` (default `busybox:1.37`, for reading the host's addresses) (see [Networking](#networking)). - `github_runner_cluster_compose_files`: extra Compose files for the host's k3s project, merged after `docker-compose.yml`, as paths relative to `github_runner_cluster_dir` or absolute. Several nodes on one Docker host each need their own `github_runner_cluster_dir` and `github_runner_cluster_compose_project`. - `github_runner_cluster_wait_for_nodes`: wait for every node to be Ready afterwards, clearing stale node-password registrations if a node fails to rejoin (installs the kubernetes Python client through `github_runner_k8s_client`). @@ -179,4 +243,4 @@ or read from 1Password by adding `exadev.github_runner.github_runner_secrets_one `tests/litestream/run.sh` runs the `headscale_hosted` provider with Litestream and a warm standby against an S3-compatible store on this machine's Docker, registers a real Tailscale client, promotes the standby and checks that users, nodes and pre-auth keys survive, that the client reconnects and that the promoted server keeps replicating. It also checks that `headscale_in_cluster` refuses to run without Litestream. `.github/workflows/litestream.yml` runs it on pull requests that touch the Headscale providers. -`tests/mesh/run.sh` brings up three k3s nodes in Docker on one machine and joins them through Headscale with this role, for the `headscale_hosted` and `headscale_in_cluster` providers, and checks the `headscale_existing` policy check against a real server. Each cluster scenario runs the role a second time and fails if any node container restarted. The `compose_faulty` scenario checks the role refuses a Compose release that has that fault. With `GRTEST_PIN_CONTEXT=1` the cluster scenarios pin `github_runner_cluster_docker_context` while the current Docker context the playbooks see points at a socket that does not exist, so any docker call that ignored the pin fails, and they check the current context is unchanged afterwards; `tests/litestream/run.sh` takes the same setting for the Headscale hosts it delegates to and the promotion playbook. The `context_checks` scenario checks the role refuses a pinned context that does not exist, and one set alongside `DOCKER_HOST`, before writing anything. `.github/workflows/mesh-integration.yml` runs them on pull requests that touch the role, the hosted scenario on 2.39.0, the first fixed release, as well as on a current one. +`tests/mesh/run.sh` brings up three k3s nodes in Docker on one machine and joins them through Headscale with this role, for the `headscale_hosted` and `headscale_in_cluster` providers, and checks the `headscale_existing` policy check against a real server. Each cluster scenario runs the role a second time and fails if any node container restarted. The `compose_faulty` scenario checks the role refuses a Compose release that has that fault. With `GRTEST_PIN_CONTEXT=1` the cluster scenarios pin `github_runner_cluster_docker_context` while the current Docker context the playbooks see points at a socket that does not exist, so any docker call that ignored the pin fails, and they check the current context is unchanged afterwards; `tests/litestream/run.sh` takes the same setting for the Headscale hosts it delegates to and the promotion playbook. The `context_checks` scenario checks the role refuses a pinned context that does not exist, and one set alongside `DOCKER_HOST`, before writing anything. The `host_network` scenario (Linux only) gives each node a Docker daemon of its own at a fixed address on the test network, standing in for a VM with an address on the LAN, runs the role with `github_runner_cluster_network_mode: host`, and checks that every node's Tailscale client reaches every other directly at that address and that no port is published. `tests/unit/test_cluster_network.py` checks the rendered Compose project in both modes and the role's list of Compose files; `tests/unit/test_cluster_guard.py` checks the address and quorum filters, and `tests/unit/test_host_network_role.py` runs the role's `tasks/host_network.yml` against stand-in `docker` and `kubectl` commands. `.github/workflows/mesh-integration.yml` runs them on pull requests that touch the role, the hosted scenario on 2.39.0, the first fixed release, as well as on a current one. diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index afa370c..2ba0198 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -113,13 +113,20 @@ github_runner_cluster_headscale_standby_host: "" # headscale_in_cluster keeps Headscale's state on the bootstrap server's k3s data volume, which a static pod cannot replace with a replicated persistent volume (kubelet runs it with no API server, so it cannot bind a claim), so the provider requires Litestream. Set this to true to run without it and accept that losing that volume loses every registration. github_runner_cluster_headscale_accept_node_local_state: false +# The network the node's k3s container runs on. bridge (the default) is Compose's own bridge network, behind the Docker host's NAT, with the API server's port published. host puts the container on the Docker host's own network (on macOS, the Docker VM's, which then needs an address of its own on the LAN; see the role README's Networking section): its Tailscale client then offers the host's own addresses to its peers, so nodes that share a network connect directly instead of through a relay, and fall back to a relay or a hole-punched path when they do not. Nothing k3s uses changes (node IP, etcd peer URLs and certificate names stay on the mesh), so a host can move between the two with a container recreate. With host, k3s's ports, routes and firewall rules (kube-proxy, flannel, Tailscale's own and the entrypoint's unreachable route for the mesh ranges) live in the Docker host's namespace, shared with every other container on the host network there, and the API server (6443) and kubelet (10250) answer on every address the host has, its LAN address included. One node per Docker host: the role refuses host mode where the host network already has a Tailscale device that is not the node's own, as on a Linux machine running its own Tailscale client with no VM between, or beside a second node on the same Docker host (see the role README's Networking section). Per host, in its host_vars. +github_runner_cluster_network_mode: bridge +# Required with github_runner_cluster_network_mode host: the LAN the nodes share, in CIDR notation (for example 192.0.2.0/24). Before starting the node the role checks that the Docker host's own network (on macOS, the Docker VM's) has an address in it, and fails without changing anything when it has none, since a node on the host network of a VM without a LAN address gains nothing and still loses its published port. The address is read with `ip` from a throwaway container on the host network, run from github_runner_cluster_network_probe_image. +github_runner_cluster_lan_cidr: "" +github_runner_cluster_network_probe_image: "busybox:1.37" + # Extra Compose files merged after docker-compose.yml for this host's k3s project, as paths relative to github_runner_cluster_dir or absolute. Nothing needs them in normal use; the mesh integration tests use one to put several nodes on one Docker host. github_runner_cluster_compose_files: [] -# Derived, never set directly: every Compose file for this host's k3s project, in merge order: docker-compose.yml, then the override the headscale_in_cluster provider writes for the bootstrap server, then github_runner_cluster_compose_files. With no files beyond docker-compose.yml the role passes none at all and Compose reads it by itself, exactly as a plain `docker compose up` in github_runner_cluster_dir would. +# Derived, never set directly: every Compose file for this host's k3s project, in merge order: docker-compose.yml, then the override the headscale_in_cluster provider writes for the bootstrap server, then github_runner_cluster_compose_files, then, with github_runner_cluster_network_mode host, docker-compose.host-network.yml, last so that its emptied port list wins over any file that publishes one. With no files beyond docker-compose.yml the role passes none at all and Compose reads it by itself, exactly as a plain `docker compose up` in github_runner_cluster_dir would. github_runner_cluster_compose_file_list: >- {{ ['docker-compose.yml'] + (['docker-compose.mesh.yml'] if github_runner_cluster_mesh == 'headscale_in_cluster' and github_runner_cluster_effective_bootstrap | default(false) | bool else []) - + github_runner_cluster_compose_files }} + + github_runner_cluster_compose_files + + (['docker-compose.host-network.yml'] if github_runner_cluster_network_mode == 'host' else []) }} # The Docker context this host's node runs in, pinned by name (for example "colima"). Empty, the default, follows the host's current Docker context, or DOCKER_HOST when that is set. When set, every community.docker module call in the role passes it as cli_context and every docker CLI command runs with DOCKER_CONTEXT set to it, so both always reach the same daemon, whatever the host's current context is at the time. The role fails before touching the host if the context does not exist there, or if DOCKER_HOST is set as well, and it never changes the host's current context. Set it on any host with more than one Docker engine installed: on a Mac running its node in Colima with Docker Desktop also installed, merely launching Docker Desktop makes desktop-linux the current context, and an unpinned run would then build a second, empty node in Docker Desktop instead of managing the real one. Per host, like every value here that names something on the host; the Headscale tasks read it from the inventory of the host they act on. github_runner_cluster_docker_context: "" # The node's k3s container, as docker-compose.yml names it. The headscale_in_cluster provider runs commands inside it; override it only alongside an extra Compose file that renames the container. diff --git a/roles/github_runner_cluster/files/docker-compose.host-network.yml b/roles/github_runner_cluster/files/docker-compose.host-network.yml new file mode 100644 index 0000000..b2f3173 --- /dev/null +++ b/roles/github_runner_cluster/files/docker-compose.host-network.yml @@ -0,0 +1,9 @@ +# Merged last, after docker-compose.yml and every other override, when github_runner_cluster_network_mode is host. The node's container then shares the Docker host's own network namespace (on macOS, the Docker VM's) instead of sitting on a Compose bridge network behind its NAT. Its Tailscale client sees the host's own addresses and offers them to its peers as direct endpoints, so nodes on one LAN, each with an address of its own on it, connect directly; any other pair falls back to a relay or a hole-punched path, as on a bridge network. Nothing k3s uses changes: the node IP, etcd's peer URLs and the certificates' names stay the mesh's. +# +# ports is emptied because a container on the host network publishes nothing: every port it listens on, the API server's included, is already the host's. On a Docker VM the VM's own port forwarding (Lima's, for Colima) then makes the API server reach the host's loopback as a published port did. +services: + k3s: + network_mode: host + ports: !reset [] + k3s-agent: + network_mode: host diff --git a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh index 4b1f5df..acf060a 100755 --- a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh +++ b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh @@ -40,7 +40,7 @@ fi # --resolv-conf points kubelet at a synthetic file containing only Tailscale's own fixed, well-known MagicDNS resolver IP (100.100.100.100, always this value, regardless of tailnet or control server), with no search domains at all. Without this, kubelet's dnsPolicy: ClusterFirst merges the container's own /etc/resolv.conf search list into every pod's resolv.conf, and that file is owned and overwritten by `tailscale up` itself to include the tailnet's configured search domains. Confirmed live: a personal domain configured as a global Tailscale DNS search path carried a wildcard record pointing at a Netlify-hosted site, and Kubernetes' ndots:5 default made a pod's resolver try every search-suffixed name (e.g. "api.github.com.") before the bare hostname, so curl connected to Netlify's edge and TLS hostname verification failed. CoreDNS itself was never affected (it forwards queries verbatim with no ndots/search logic of its own) and still resolves every name through the real Tailscale resolver; this flag only stops kubelet copying the search list into pods. printf 'nameserver 100.100.100.100\n' > /etc/k3s-resolv.conf -# Brings Tailscale up ourselves, ahead of k3s's own --vpn-auth doing the same thing, to learn this node's Tailscale IP for --node-ip before starting k3s. Confirmed against k3s's own vpn.go source: StartVPN checks the backend state first and skips its own `tailscale up` once the state is already "Running", so k3s's own call becomes a no-op rather than a second registration. +# Brings Tailscale up ourselves, ahead of k3s's own --vpn-auth doing the same thing, to learn this node's Tailscale IP for --node-ip before starting k3s. k3s's StartVPN (pkg/vpn/vpn.go) then runs `tailscale up --authkey ... --timeout=30s --reset` on every start, adding only --login-server and whatever follows ",extraArgs=" in --vpn-auth. An already-authenticated daemon keeps its identity through that, so it does not register a second device, but --reset returns every `tailscale up` or `tailscale set` preference not passed again to its default. A Tailscale setting this node needs must therefore be a tailscaled daemon flag (as on the tailscaled line above) or go through --vpn-auth's extraArgs, never a separate `tailscale set`. # # Only calls `tailscale up` if the daemon isn't already authenticated from its persisted state volume. Confirmed live: calling it unconditionally on every container restart re-registered a brand new device each time under the reusable join key instead of reconnecting the existing one, leaving a growing trail of stale devices on the tailnet. A persisted login needs a moment to reconnect after tailscaled starts (its backend passes through NoState and Starting first), so this waits for the backend to settle before deciding; registering again during that window re-authenticates a node that was about to reconnect by itself. backend_state() { diff --git a/roles/github_runner_cluster/tasks/host_network.yml b/roles/github_runner_cluster/tasks/host_network.yml new file mode 100644 index 0000000..0a4b028 --- /dev/null +++ b/roles/github_runner_cluster/tasks/host_network.yml @@ -0,0 +1,183 @@ +--- +# Before the role changes anything on the host: github_runner_cluster_network_mode must be bridge or host; with host, the Docker host must have an address on the LAN the nodes share (github_runner_cluster_lan_cidr) and no Tailscale device of another client on its network; and whenever the node's running container is on the other kind of network, Compose is about to recreate it, so the role first refuses unless the cluster stays safe without this node (plugins/filter/cluster_guard.py's cluster_restart_blockers): the other servers with a Ready node and a healthy etcd must still be a majority of all servers, and no ARC runner pod may be on this node. Whatever cannot be read refuses the move. Every task here only reads (the probe pulls its small image if the host lacks it); a failure changes nothing on the host. + +- name: Fail if the node's network mode is not one the role knows + ansible.builtin.fail: + msg: >- + github_runner_cluster_network_mode is '{{ github_runner_cluster_network_mode }}'; it must be bridge (Compose's own + network, the default) or host (the Docker host's network). See the role's defaults for what each does. + when: github_runner_cluster_network_mode not in ['bridge', 'host'] + +- name: Fail if the host network is asked for without the LAN it should be on + ansible.builtin.fail: + msg: >- + github_runner_cluster_network_mode is host for {{ inventory_hostname }}, but github_runner_cluster_lan_cidr is empty. + Set it to the LAN the nodes share, in CIDR notation (for example 192.0.2.0/24), so the role can check the Docker host + has an address on it. + when: + - github_runner_cluster_network_mode == 'host' + - github_runner_cluster_lan_cidr | trim | length == 0 + +- name: Read whether the node's container runs, and on which network + # docker ps rather than docker inspect, so that a container that does not exist (a first run) reads as absent while a daemon that cannot be reached fails the run. + ansible.builtin.command: + argv: + - docker + - ps + - --all + - --filter + - "name=^/{{ github_runner_cluster_network_container }}$" + - --format + - "{{ '{{' }}.State{{ '}}' }} {{ '{{' }}.Networks{{ '}}' }}" + environment: "{{ github_runner_cluster_docker_cli_environment }}" + vars: + github_runner_cluster_network_container: >- + {{ github_runner_cluster_container_name if github_runner_cluster_effective_node_role == 'server' else 'github-runner-k3s-agent' }} + register: github_runner_cluster_node_container + changed_when: false + check_mode: false + +- name: Note whether the node's container runs, and whether on the host network + # A container on the host network lists exactly one network, "host". + ansible.builtin.set_fact: + github_runner_cluster_network_running: "{{ (github_runner_cluster_node_container.stdout.split() | first | default('')) == 'running' }}" + github_runner_cluster_network_on_host: "{{ (github_runner_cluster_node_container.stdout.split()[1:] | join(' ')) == 'host' }}" + +- name: Note whether Compose is about to move a running node to the other network + # Only when the running container is on the other kind of network than the one asked for, which is when Compose recreates it. + ansible.builtin.set_fact: + github_runner_cluster_network_switching: >- + {{ github_runner_cluster_network_running | bool + and (github_runner_cluster_network_on_host | bool) != (github_runner_cluster_network_mode == 'host') }} + +- name: Read the Docker host's own IPv4 addresses and network devices + # From a throwaway container on the host network: on macOS that is the Docker VM's network, which is where the node's Tailscale client looks for addresses to offer. Runs under --check too, since the checks below depend on it. + ansible.builtin.command: + argv: [docker, run, --rm, --network, host, "{{ github_runner_cluster_network_probe_image }}", sh, -c, "ip -4 -o addr show && echo -- && ip -o link show"] + environment: "{{ github_runner_cluster_docker_cli_environment }}" + register: github_runner_cluster_host_network + changed_when: false + check_mode: false + when: github_runner_cluster_network_mode == 'host' + +- name: Fail if the Docker host has no address on the LAN + vars: + github_runner_cluster_host_addresses: "{{ github_runner_cluster_host_network.stdout.split('\n--\n') | first | split('\n') }}" + ansible.builtin.fail: + msg: >- + The Docker host {{ inventory_hostname }}'s node would run on has no address in {{ github_runner_cluster_lan_cidr }} + (its addresses: {{ github_runner_cluster_host_addresses | map('regex_search', 'inet [0-9./]+') | select | map('regex_replace', '^inet ', '') | join(', ') }}), + so on the host network its Tailscale client would have nothing to offer its peers on the LAN. On macOS the Docker VM + needs an address of its own on the LAN, which a Colima profile has only when it was created bridged (see the role + README's Networking section for moving a node to one). Nothing was changed. + when: + - github_runner_cluster_network_mode == 'host' + - github_runner_cluster_host_addresses | exadev.github_runner.addresses_in_network(github_runner_cluster_lan_cidr) | length == 0 + +- name: Fail if another Tailscale client already owns a device on the Docker host's network + # The node's tailscaled creates tailscale0 and Tailscale's routing table, rules and firewall chains in the namespace it runs in. On the host network that namespace is the Docker host's: on a Linux machine with no VM, the machine's own, where its own Tailscale client may already run; on any Docker host, where a second node on the host network already runs one. Two clients there fight over the device and the routes. The node's own device is expected once it runs on the host network. + vars: + github_runner_cluster_host_devices: "{{ github_runner_cluster_host_network.stdout.split('\n--\n') | last | split('\n') | select('search', '^[0-9]+: tailscale[0-9]+[:@]') | list }}" + ansible.builtin.fail: + msg: >- + The Docker host {{ inventory_hostname }}'s node would run on already has a Tailscale device on its network + ({{ github_runner_cluster_host_devices | map('regex_replace', '^[0-9]+: ([^:@]+).*$', '\1') | join(', ') }}), which belongs to + another Tailscale client: the host's own (on a Linux machine with no VM, the host network is the machine's own) or + another node on the host network of the same Docker host. A node there would fight it over the device and the routes, + so host mode needs a Docker host (a VM) of its own for the node. Nothing was changed. + when: + - github_runner_cluster_network_mode == 'host' + - not (github_runner_cluster_network_running | bool and github_runner_cluster_network_on_host | bool) + - github_runner_cluster_host_devices | length > 0 + +- name: Check that recreating the node on the other network is safe now + when: github_runner_cluster_network_switching | bool + block: + - name: Refuse to move more than one node between networks in one run + # Each host's checks read the cluster before any node is recreated, so two hosts moving in the same run would each count the other as healthy and could be recreated together. + vars: + github_runner_cluster_network_movers: >- + {% for host in ansible_play_hosts if hostvars[host].github_runner_cluster_network_switching | default(false) | bool %}{{ host }} {% endfor %} + ansible.builtin.fail: + msg: >- + This run would move {{ github_runner_cluster_network_movers | trim | split | join(', ') }} to another network at once, each + recreating its node. Move one node at a time (--limit to one host). Nothing was changed. + when: github_runner_cluster_network_movers | trim | split | length > 1 + + - name: Read the cluster's nodes and this node's pods through each server of the inventory + # Through the image's kubectl in each server's own container (as running_version.yml reads the cluster), this one first, each on its own host with its pinned Docker context, so the read needs no kubeconfig or published port on the host and no particular server to be up; each read is bounded by github_runner_cluster_cluster_read_timeout as a whole and as kubectl's --request-timeout. The first server that answers is the one the checks below use; plugins/filter/cluster_guard.py's cluster_guard_state says why for each when none does. + ansible.builtin.shell: | + set -eu + nodes=$(docker exec "$container" kubectl get nodes --request-timeout="${timeout}s" -o json) + pods=$(docker exec "$container" kubectl get pods --all-namespaces --request-timeout="${timeout}s" --field-selector "spec.nodeName=${node}" -o json) + printf '{"nodes": %s, "pods": %s}\n' "$nodes" "$pods" + environment: >- + {{ ({'DOCKER_CONTEXT': github_runner_cluster_guard_context} if github_runner_cluster_guard_context | length > 0 else {}) + | combine({'container': github_runner_cluster_guard_container, 'node': github_runner_cluster_effective_node_name, + 'timeout': github_runner_cluster_cluster_read_timeout | string}) }} + delegate_to: "{{ item }}" + loop: >- + {{ ([inventory_hostname] if github_runner_cluster_effective_node_role == 'server' else []) + + (github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') + | map(attribute='key') | reject('equalto', inventory_hostname) | list) }} + vars: + github_runner_cluster_guard_container: >- + {{ github_runner_cluster_container_name if item == inventory_hostname + else hostvars[item].github_runner_cluster_container_name | default('github-runner-k3s') }} + github_runner_cluster_guard_context: >- + {{ (github_runner_cluster_docker_context | default('') if item == inventory_hostname + else hostvars[item].github_runner_cluster_docker_context | default('')) | trim }} + register: github_runner_cluster_guard_reads + changed_when: false + failed_when: false + ignore_unreachable: true + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + + - name: Fail if no server of the inventory could be read + vars: + github_runner_cluster_guard_read: "{{ github_runner_cluster_guard_reads.results | exadev.github_runner.cluster_guard_state }}" + ansible.builtin.fail: + msg: >- + Not moving {{ github_runner_cluster_effective_node_name }} to the {{ github_runner_cluster_network_mode }} network, + which recreates its container: {{ github_runner_cluster_guard_read.problem }} Nothing was changed. + when: github_runner_cluster_guard_read.state is none + + - name: Read etcd's health through every other server's own API server + # kubectl inside each server's container asks that server's API server, whose /readyz/etcd answers "ok" only while its etcd serves; on that host, with its pinned Docker context. Bounded like the reads above by github_runner_cluster_cluster_read_timeout, as a whole and as the API call's --request-timeout. A host that cannot be reached, or a read that fails or runs out of time, counts as unhealthy rather than failing here, so the refusal below names it. + ansible.builtin.command: + argv: [docker, exec, "{{ hostvars[item.key].github_runner_cluster_container_name | default('github-runner-k3s') }}", kubectl, get, "--request-timeout={{ github_runner_cluster_cluster_read_timeout | int }}s", --raw, /readyz/etcd] + environment: >- + {{ {'DOCKER_CONTEXT': hostvars[item.key].github_runner_cluster_docker_context} + if hostvars[item.key].github_runner_cluster_docker_context | default('') | length > 0 else {} }} + delegate_to: "{{ item.key }}" + loop: >- + {{ github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') + | rejectattr('key', 'equalto', inventory_hostname) | list }} + loop_control: + label: "{{ item.value.node_name }}" + register: github_runner_cluster_guard_etcd + changed_when: false + failed_when: false + ignore_unreachable: true + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + when: github_runner_cluster_effective_node_role == 'server' + + - name: Refuse to recreate the node on the other network while the cluster needs it + vars: + github_runner_cluster_guard: "{{ (github_runner_cluster_guard_reads.results | exadev.github_runner.cluster_guard_state).state }}" + github_runner_cluster_guard_blockers: >- + {{ github_runner_cluster_guard.nodes + | exadev.github_runner.cluster_restart_blockers(github_runner_cluster_guard.pods, + github_runner_cluster_effective_node_name, + github_runner_cluster_topology.hosts | dict2items + | selectattr('value.role', 'equalto', 'server') + | map(attribute='value.node_name') | list, + github_runner_cluster_guard_etcd.results | default([])) }} + ansible.builtin.fail: + msg: >- + Not moving {{ github_runner_cluster_effective_node_name }} to the {{ github_runner_cluster_network_mode }} network, + which recreates its container: {{ github_runner_cluster_guard_blockers | join(' ') }} Nothing was changed; run again + once that has cleared. + when: github_runner_cluster_guard_blockers | length > 0 diff --git a/roles/github_runner_cluster/tasks/main.yml b/roles/github_runner_cluster/tasks/main.yml index 2d57220..b4362c8 100644 --- a/roles/github_runner_cluster/tasks/main.yml +++ b/roles/github_runner_cluster/tasks/main.yml @@ -40,6 +40,10 @@ - name: Check the k3s version against what this node and the cluster run now ansible.builtin.include_tasks: running_version.yml +- name: Check the node's network mode, the host network's preconditions, and that moving the node between networks is safe now + # Before anything changes on the host, and so before the mesh step, which starts the bootstrap server's Compose project itself with the headscale_in_cluster provider. + ansible.builtin.include_tasks: host_network.yml + - name: Fail if the k3s token is not set ansible.builtin.fail: msg: >- @@ -66,6 +70,7 @@ mode: "{{ item.mode }}" loop: - {src: docker-compose.yml, mode: "0644"} + - {src: docker-compose.host-network.yml, mode: "0644"} - {src: k3s/Dockerfile, mode: "0644"} - {src: k3s/node-entrypoint.sh, mode: "0755"} @@ -130,6 +135,21 @@ - name: End the role here, since an agent host has no API of its own to wait on ansible.builtin.meta: end_role +- name: Wait for the API server to answer on this host's loopback, with the node on the host network + # With github_runner_cluster_network_mode host nothing publishes the API server's port: the kubeconfig k3s writes (https://127.0.0.1:) reaches it only if the port is the host's own, or, on a Docker VM, if the VM forwards it to the host as Lima does for Colima. Checked only when the role goes on to use that kubeconfig from the host, so a failure names the cause instead of surfacing as an unreachable API in a later task. + ansible.builtin.wait_for: + host: 127.0.0.1 + port: "{{ github_runner_cluster_api_port }}" + timeout: 300 + msg: >- + Nothing answers on 127.0.0.1:{{ github_runner_cluster_api_port }} on {{ inventory_hostname }}, so the kubeconfig k3s writes + cannot reach the API server. With github_runner_cluster_network_mode host the port is not published; on a Docker VM it + must be forwarded from the VM to the host (Colima forwards a port a container on the VM's host network listens on). Check + the VM's port forwarding, or set github_runner_cluster_network_mode back to bridge. + when: + - github_runner_cluster_network_mode == 'host' + - github_runner_cluster_wait_for_nodes | bool or github_runner_cluster_record_deployed_version | bool + - name: Give this host the kubernetes Python client before waiting on the API ansible.builtin.include_role: name: github_runner_k8s_client diff --git a/tests/cluster_network/compose_files.yml b/tests/cluster_network/compose_files.yml new file mode 100644 index 0000000..00420b1 --- /dev/null +++ b/tests/cluster_network/compose_files.yml @@ -0,0 +1,14 @@ +--- +# Writes the cluster role's derived Compose file list (github_runner_cluster_compose_file_list, from the role's defaults) to compose_files_output as JSON, for tests/unit/test_cluster_network.py. Role variables are passed as extra variables, which outrank the defaults loaded here. +- name: Write the cluster role's Compose file list + hosts: localhost + connection: local + gather_facts: false + vars_files: + - ../../roles/github_runner_cluster/defaults/main.yml + tasks: + - name: Write the list + ansible.builtin.copy: + content: "{{ github_runner_cluster_compose_file_list | to_json }}" + dest: "{{ compose_files_output }}" + mode: "0644" diff --git a/tests/host_network/play.yml b/tests/host_network/play.yml new file mode 100644 index 0000000..e624995 --- /dev/null +++ b/tests/host_network/play.yml @@ -0,0 +1,14 @@ +--- +# Runs the cluster role's tasks/host_network.yml on its own, for tests/unit/test_host_network_role.py, against stand-in docker and kubectl commands the test puts first on PATH, on the hosts of an inventory the test writes (all on a local connection, so a task delegated to another server runs here too). Takes the role variables the task file reads as play vars below and extra variables. +- name: Run the host network checks + hosts: all + connection: local + gather_facts: false + vars: + github_runner_cluster_docker_cli_environment: {} + github_runner_cluster_container_name: github-runner-k3s + github_runner_cluster_network_probe_image: "busybox:1.37" + github_runner_cluster_cluster_read_timeout: 30 + tasks: + - name: Include the role's host network checks + ansible.builtin.include_tasks: ../../roles/github_runner_cluster/tasks/host_network.yml diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 75370c6..734ddd7 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. +# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). host_network each node runs in a Docker daemon of its own (a docker:dind container standing in for a VM with an address on the LAN, on the shared test network) with github_runner_cluster_network_mode host, joined through a Headscale server the test runs (headscale_existing); the nodes' Tailscale clients must then reach each other directly at those daemons' addresses, and the role must not publish any port; then the role must refuse to move two running nodes to the bridge network in one run, and move one there and back through its guard, the node keeping its mesh address and etcd membership. Linux only, since the control node reaches each daemon at its address on the test network. The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. # # With GRTEST_PIN_CONTEXT=1, the cluster scenarios run the role, and the in_cluster recovery playbook, with github_runner_cluster_docker_context pinned while the current Docker context they see points at nothing (tests/lib/docker_context.sh), so any docker call that ignored the pin fails the run; afterwards the current context must be unchanged. context_checks the role refuses a pinned context that does not exist, and a pinned context alongside DOCKER_HOST, before writing anything (not in the default set). # @@ -16,6 +16,10 @@ subnet_prefix=172.31.250 headscale_ip="${subnet_prefix}.10" node_count=3 etcd_image=gcr.io/etcd-development/etcd:v3.5.21 +# The Docker daemon each host_network node runs in, standing in for a VM with an address on the LAN. +dind_image=docker:27.5.1-dind +# Set by scenario_host_network: every node's containers are then on that node's own daemon, reached through a Docker context per node. +host_network=0 busybox_image=busybox:1.37 # Tries per pod pair, 5 seconds apart plus each try's own timeout: about two minutes for the mesh to converge after a restart. reach_attempts=8 @@ -35,11 +39,11 @@ fail() { diagnose() { local index for index in $(seq 1 "$node_count"); do - docker inspect "$(container "$index")" >/dev/null 2>&1 || continue + docker_on "$index" inspect "$(container "$index")" >/dev/null 2>&1 || continue echo "----- $(container "$index"): tailscale status" - docker exec "$(container "$index")" tailscale status &1 | head -n 20 || true + docker_on "$index" exec "$(container "$index")" tailscale status &1 | head -n 20 || true echo "----- $(container "$index"): last log lines" - docker logs --tail 60 "$(container "$index")" 2>&1 | cut -c1-400 || true + docker_on "$index" logs --tail 60 "$(container "$index")" 2>&1 | cut -c1-400 || true done if docker inspect grtest-headscale >/dev/null 2>&1; then echo "----- grtest-headscale: nodes and routes" @@ -58,6 +62,14 @@ fi node_name() { echo "grtest-node$1"; } node_ip() { echo "${subnet_prefix}.$((10 + $1))"; } container() { echo "grtest-node$1-k3s"; } +vm() { echo "grtest-vm$1"; } + +# The docker CLI for node $1's daemon: the machine's own, or with host_network that node's own daemon through its context. +docker_on() { + local index="$1" + shift + if [ "$host_network" = 1 ]; then docker --context "$(vm "$index")" "$@"; else docker "$@"; fi +} teardown() { for index in $(seq 1 "$node_count"); do @@ -69,6 +81,10 @@ teardown() { docker compose --project-directory "${work}/headscale" -f "${work}/headscale/docker-compose.yml" down -v >/dev/null 2>&1 || true fi docker rm -f grtest-headscale grtest-hs-existing grtest-etcdctl >/dev/null 2>&1 || true + for index in $(seq 1 "$node_count"); do + docker rm -f -v "$(vm "$index")" >/dev/null 2>&1 || true + docker context rm -f "$(vm "$index")" >/dev/null 2>&1 || true + done docker network rm "$network" >/dev/null 2>&1 || true rm -rf "$work" } @@ -185,7 +201,7 @@ run_role() { if [ "$pin_context" = 1 ]; then grtest_assert_current_context_unchanged "$docker_config" || fail "the role changed the current Docker context"; fi } -kubectl_on() { docker exec -i "$(container "$1")" kubectl "${@:2}"; } +kubectl_on() { docker_on "$1" exec -i "$(container "$1")" kubectl "${@:2}"; } assert_cluster() { local expected_members nodes members index other ip route @@ -200,8 +216,8 @@ assert_cluster() { log "Checking the servers were chosen automatically (all three) and etcd has three members" nodes=$(kubectl_on 1 get nodes -l node-role.kubernetes.io/etcd=true --no-headers | wc -l | tr -d ' ') [ "$nodes" = "$node_count" ] || fail "expected ${node_count} etcd nodes, found ${nodes}" - docker rm -f grtest-etcdctl >/dev/null 2>&1 || true - members=$(docker run --rm --name grtest-etcdctl --network "container:$(container 1)" --volumes-from "$(container 1)" --entrypoint etcdctl "$etcd_image" \ + docker_on 1 rm -f grtest-etcdctl >/dev/null 2>&1 || true + members=$(docker_on 1 run --rm --name grtest-etcdctl --network "container:$(container 1)" --volumes-from "$(container 1)" --entrypoint etcdctl "$etcd_image" \ --endpoints https://127.0.0.1:2379 --cacert /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \ --cert /var/lib/rancher/k3s/server/tls/etcd/client.crt --key /var/lib/rancher/k3s/server/tls/etcd/client.key \ member list -w simple) @@ -251,7 +267,7 @@ EOF for index in $(seq 1 "$node_count"); do while IFS=, read -r _ other ip; do [ "$other" = "$(node_name "$index")" ] && continue - route=$(docker exec "$(container "$index")" ip route get "$ip" /dev/null + docker context create "$(vm "$index")" --docker "host=tcp://$(node_ip "$index"):2375" >/dev/null + done + for index in $(seq 1 "$node_count"); do + local deadline=$((SECONDS + 120)) + until docker --context "$(vm "$index")" info >/dev/null 2>&1; do + [ "$SECONDS" -lt "$deadline" ] || fail "the Docker daemon in $(vm "$index") did not start" + sleep 1 + done + done +} + +# A Headscale server on this machine's Docker, at a fixed address on the test network, with a policy that lets the nodes register, reach each other and route pods; prints the API key and a join key on two lines. +start_headscale_for_host_network() { + local fixtures="${repo_root}/tests/mesh/fixtures/headscale-existing" + mkdir -p "${work}/hs-host-network" + sed -e "s#^server_url: .*#server_url: http://${headscale_ip}:8080#" -e 's#^ base_domain: .*# base_domain: mesh.grtest.internal#' \ + "${fixtures}/config.yaml" > "${work}/hs-host-network/config.yaml" + cat > "${work}/hs-host-network/policy.hujson" <<'POLICY' +{ + "tagOwners": {"tag:github-runner-node": []}, + "autoApprovers": {"routes": {"10.42.0.0/16": ["tag:github-runner-node"]}}, + "grants": [{"src": ["tag:github-runner-node", "10.42.0.0/16"], "dst": ["tag:github-runner-node", "10.42.0.0/16"], "ip": ["*"]}], +} +POLICY + docker run -d --name grtest-hs-existing --network "$network" --ip "$headscale_ip" \ + -v "${work}/hs-host-network:/etc/headscale:ro" "headscale/headscale:v$(sed -n 's/^github_runner_cluster_headscale_version: "\(.*\)"$/\1/p' "${repo_root}/roles/github_runner_cluster/defaults/main.yml")" serve >/dev/null + until docker exec grtest-hs-existing headscale health >/dev/null 2>&1; do sleep 1; done + docker exec grtest-hs-existing headscale apikeys create --expiration 1d | tail -n 1 + # 90 days: the headscale_existing provider refuses a join key that expires within 14. + docker exec grtest-hs-existing headscale preauthkeys create --reusable --tags tag:github-runner-node --expiration 90d | tail -n 1 +} + +write_host_network_inventory() { + local api_key="$1" join_key="$2" index + for index in $(seq 1 "$node_count"); do + mkdir -p "${work}/$(node_name "$index")" + printf 'services:\n k3s:\n container_name: %s\n' "$(container "$index")" > "${work}/$(node_name "$index")/compose.grtest.yml" + done + { + echo "all:" + echo " vars:" + echo " ansible_connection: local" + echo " ansible_python_interpreter: $(command -v python3)" + echo " children:" + echo " github_runner_cluster:" + echo " vars:" + echo " github_runner_cluster_mesh: headscale_existing" + echo " github_runner_cluster_tailnet_domain: mesh.grtest.internal" + echo " github_runner_cluster_headscale_url: http://${headscale_ip}:8080" + echo " github_runner_cluster_headscale_api_key: ${api_key}" + echo " github_runner_cluster_headscale_join_key: ${join_key}" + echo " github_runner_cluster_compose_files: [compose.grtest.yml]" + echo " github_runner_cluster_network_mode: host" + echo " github_runner_cluster_lan_cidr: ${subnet_prefix}.0/24" + echo " github_runner_cluster_k3s_token: ${k3s_token}" + echo " hosts:" + for index in $(seq 1 "$node_count"); do + echo " $(node_name "$index"):" + echo " github_runner_cluster_dir: ${work}/$(node_name "$index")" + echo " github_runner_cluster_compose_project: $(node_name "$index")" + echo " github_runner_cluster_container_name: $(container "$index")" + echo " github_runner_cluster_docker_context: $(vm "$index")" + done + } > "${work}/inventory.yml" +} + +# With host networking, every node's Tailscale client must reach every other node directly at that node's daemon's own address (its stand-in LAN address), not through a relay, and the node container must publish no port. +assert_direct_paths() { + local index other status deadline + for index in $(seq 1 "$node_count"); do + [ "$(docker_on "$index" inspect --format '{{.HostConfig.NetworkMode}}' "$(container "$index")")" = host ] || fail "$(container "$index") is not on the host network" + [ -z "$(docker_on "$index" port "$(container "$index")")" ] || fail "$(container "$index") publishes ports: $(docker_on "$index" port "$(container "$index")")" + for other in $(seq 1 "$node_count"); do + [ "$index" = "$other" ] && continue + deadline=$((SECONDS + 120)) + # tailscale ping keeps pinging until a direct path answers, or gives up and says the connection is relayed. + until status=$(docker_on "$index" exec "$(container "$index")" tailscale ping --c 10 "$(node_name "$other")" &1) && echo "$status" | grep -q "via $(node_ip "$other"):"; do + [ "$SECONDS" -lt "$deadline" ] || fail "$(node_name "$index") reaches $(node_name "$other") only indirectly: ${status}" + sleep 5 + done + echo " $(node_name "$index") -> $(node_name "$other"): $(echo "$status" | tail -n 1)" + done + done + log "Every node reaches every other directly at its daemon's address" +} + +scenario_host_network() { + local keys api_key join_key before after + [ "$(uname -s)" = Linux ] || fail "the host_network scenario reaches each node's Docker daemon at its address on the test network, which only Linux routes to" + host_network=1 + reset_environment + start_vms + keys=$(start_headscale_for_host_network) + api_key=$(echo "$keys" | sed -n 1p) + join_key=$(echo "$keys" | sed -n 2p) + write_host_network_inventory "$api_key" "$join_key" + run_role + assert_cluster + assert_direct_paths + log "Rerunning the role, which must not restart any node" + before=$(started_at) + run_role + after=$(started_at) + [ "$before" = "$after" ] || fail "rerunning the role restarted a node container: ${before} -> ${after}" + move_between_networks +} + +# After node $1's container was recreated, the API can still show its echo pod at an address its new sandbox no longer has; replacing the pod makes assert_cluster's rollout wait for the node's kubelet and probe the pod's real address. +replace_echo_pod() { + kubectl_on 1 delete pod -l app=grtest-echo --field-selector "spec.nodeName=$(node_name "$1")" --timeout=300s || fail "the echo pod on $(node_name "$1") was not replaced" +} + +# Moves running nodes between the host and the bridge network through the role, so its guard reads a real cluster: two nodes in one run must be refused before anything changes, and one node must move to the bridge network and back keeping its mesh address and etcd membership. +move_between_networks() { + local output before mesh_address + log "Moving two nodes to the bridge network in one run must be refused before anything changes" + before=$(started_at) + if output=$(ansible_run -i "${work}/inventory.yml" "${repo_root}/tests/mesh/cluster.yml" --limit "$(node_name 2),$(node_name 3)" -e github_runner_cluster_network_mode=bridge 2>&1); then + echo "$output" + fail "the role moved two nodes between networks in one run" + fi + echo "$output" | grep -q "to another network at once" || { echo "$output"; fail "the role failed for another reason than moving two nodes at once"; } + [ "$(started_at)" = "$before" ] || fail "a refused move restarted a node container" + mesh_address=$(docker_on 3 exec "$(container 3)" tailscale ip -4 dict[str, Any]: + labels = {"node-role.kubernetes.io/control-plane": "true", "node-role.kubernetes.io/etcd": "true"} if server else {} + return {"metadata": {"name": name, "labels": labels}, "status": {"conditions": [{"type": "Ready", "status": "True" if ready else "Unknown"}]}} + + +def pod(name: str, node_name: str, runner: bool = True, phase: str = "Running") -> dict[str, Any]: + labels = {"actions-ephemeral-runner": "True"} if runner else {"app": "other"} + return {"metadata": {"name": name, "namespace": "runners", "labels": labels}, "spec": {"nodeName": node_name}, "status": {"phase": phase}} + + +def etcd(name: str, healthy: bool = True, unreachable: bool = False) -> dict[str, Any]: + """One loop result of the role's etcd read through a server's container.""" + if unreachable: + return {"item": {"key": f"host-{name}", "value": {"node_name": name}}, "unreachable": True, "msg": "ssh: connect to host"} + return {"item": {"key": f"host-{name}", "value": {"node_name": name}}, "rc": 0 if healthy else 1, "stdout": "ok" if healthy else "", "stderr": "" if healthy else "etcd failed: reason withheld"} + + +def blockers(nodes: list[dict[str, Any]], pods: list[dict[str, Any]], name: str, servers: list[str] | None = None, reports: list[dict[str, Any]] | None = None) -> list[str]: + """The guard with the inventory's servers defaulting to the labelled ones, and every other server's etcd healthy unless given.""" + if servers is None: + servers = [n["metadata"]["name"] for n in nodes if n["metadata"]["labels"]] + if reports is None: + reports = [etcd(server) for server in servers if server != name] + result: list[str] = guard.cluster_restart_blockers(nodes, pods, name, servers, reports) + return result + + +class ClusterRestartBlockersTest(unittest.TestCase): + def test_three_healthy_servers_and_no_runners_may_restart(self) -> None: + self.assertEqual(blockers([node("a"), node("b"), node("c")], [], "a"), []) + + def test_a_server_may_not_restart_while_another_server_is_not_ready(self) -> None: + found = blockers([node("a"), node("b"), node("c", ready=False)], [], "a") + self.assertEqual(len(found), 1) + self.assertIn("1 of 3 servers with a Ready node and a healthy etcd, fewer than the 2", found[0]) + self.assertIn("c (not Ready)", found[0]) + + def test_a_ready_server_whose_etcd_is_unhealthy_does_not_count(self) -> None: + for report in (etcd("c", healthy=False), etcd("c", unreachable=True)): + with self.subTest(report=report): + found = blockers([node("a"), node("b"), node("c")], [], "a", reports=[etcd("b"), report]) + self.assertEqual(len(found), 1) + self.assertIn("c (etcd not reported healthy)", found[0]) + + def test_a_server_whose_etcd_was_not_read_does_not_count(self) -> None: + self.assertNotEqual(blockers([node("a"), node("b"), node("c")], [], "a", reports=[etcd("b")]), []) + + def test_a_server_the_inventory_does_not_name_counts_against_the_restart(self) -> None: + self.assertEqual(blockers([node("a"), node("b"), node("c"), node("d"), node("x")], [], "a", servers=["a", "b", "c", "d"]), []) + found = blockers([node("a"), node("b"), node("x"), node("y")], [], "a", servers=["a", "b"]) + self.assertIn("x (not in the inventory, so its etcd was not read)", found[0]) + + def test_servers_the_cluster_does_not_label_refuse_the_restart(self) -> None: + unlabelled = [node("a", server=False), node("b", server=False), node("c", server=False, ready=False)] + found = blockers(unlabelled, [], "a", servers=["a", "b", "c"]) + self.assertTrue(any("does not label a, b, c as server nodes" in blocker for blocker in found), found) + + def test_a_server_missing_from_the_node_list_does_not_count(self) -> None: + found = blockers([node("a"), node("b")], [], "a", servers=["a", "b", "c"]) + self.assertIn("c (not in the node list)", found[0]) + + def test_five_servers_tolerate_one_other_unhealthy_but_not_two(self) -> None: + servers = [node(name) for name in "abcde"] + self.assertEqual(blockers([*servers[:4], node("e", ready=False)], [], "a"), []) + self.assertNotEqual(blockers([*servers[:3], node("d", ready=False), node("e", ready=False)], [], "a"), []) + + def test_this_nodes_own_readiness_does_not_count(self) -> None: + self.assertEqual(blockers([node("a", ready=False), node("b"), node("c")], [], "a"), []) + + def test_two_servers_can_never_lose_one(self) -> None: + self.assertNotEqual(blockers([node("a"), node("b")], [], "a"), []) + + def test_a_single_server_has_no_quorum_to_lose(self) -> None: + self.assertEqual(blockers([node("a"), node("w", server=False)], [], "a"), []) + + def test_an_agent_ignores_the_servers_health(self) -> None: + nodes = [node("a"), node("b", ready=False), node("c", ready=False), node("w", server=False)] + self.assertEqual(blockers(nodes, [], "w", reports=[]), []) + + def test_an_unfinished_runner_pod_on_this_node_blocks(self) -> None: + for phase in ("Running", "Pending"): + with self.subTest(phase=phase): + found = blockers([node("a"), node("b"), node("c")], [pod("runner-1", "a", phase=phase)], "a") + self.assertEqual(len(found), 1) + self.assertIn("runners/runner-1", found[0]) + + def test_finished_runners_other_pods_and_other_nodes_runners_do_not_block(self) -> None: + pods = [pod("done", "a", phase="Succeeded"), pod("failed", "a", phase="Failed"), pod("web", "a", runner=False), pod("elsewhere", "b")] + self.assertEqual(blockers([node("a"), node("b"), node("c")], pods, "a"), []) + + def test_both_reasons_are_given_together(self) -> None: + self.assertEqual(len(blockers([node("a"), node("b", ready=False), node("c")], [pod("runner-1", "a")], "a")), 2) + + def test_a_node_missing_from_the_cluster_blocks(self) -> None: + self.assertIn("not in the cluster's node list", blockers([node("b")], [], "a", servers=["a", "b"])[0]) + + def test_input_that_is_not_lists_is_an_error(self) -> None: + with self.assertRaises(guard.AnsibleFilterError): + guard.cluster_restart_blockers([], [], "a", "a", []) + + def test_the_filters_are_registered(self) -> None: + self.assertIs(guard.FilterModule().filters()["addresses_in_network"], guard.addresses_in_network) + + def test_the_restart_blockers_filter_is_registered(self) -> None: + self.assertIs(guard.FilterModule().filters()["cluster_restart_blockers"], guard.cluster_restart_blockers) + + +class ClusterGuardStateTest(unittest.TestCase): + STATE = '{"nodes": {"items": [{"metadata": {"name": "a"}}]}, "pods": {"items": []}}' + + def test_the_first_server_that_answered_gives_the_state(self) -> None: + results = [{"item": "a", "rc": 1, "stderr": "refused"}, {"item": "b", "rc": 0, "stdout": self.STATE}, {"item": "c", "rc": 0, "stdout": '{"nodes": {"items": []}, "pods": {"items": []}}'}] + self.assertEqual(guard.cluster_guard_state(results), {"state": {"nodes": [{"metadata": {"name": "a"}}], "pods": []}, "problem": ""}) + + def test_no_server_answering_says_why_for_each(self) -> None: + found = guard.cluster_guard_state([{"item": "a", "rc": 1, "stderr": "x\nrefused"}, {"item": "b", "unreachable": True}, {"item": "c", "rc": 0, "stdout": "not json"}]) + self.assertIsNone(found["state"]) + self.assertIn("a could not be read (refused); b could not be reached; c gave an answer that is not the cluster's state", found["problem"]) + + def test_a_read_cut_off_by_its_timeout_or_with_only_a_message_is_named(self) -> None: + for result in ({"item": "a", "failed": True, "timedout": {"period": 2}, "msg": "Task failed: Timed out after 2 second(s)."}, {"item": "a", "failed": True, "msg": "The ansible.builtin.shell action failed to execute in the expected time frame (2) and was terminated"}): + with self.subTest(result=result): + self.assertIn("a did not answer within github_runner_cluster_cluster_read_timeout", guard.cluster_guard_state([result])["problem"]) + self.assertIn("a could not be read (docker: command not found)", guard.cluster_guard_state([{"item": "a", "failed": True, "msg": "docker: command not found"}])["problem"]) + + def test_no_server_in_the_inventory_says_so(self) -> None: + found = guard.cluster_guard_state([]) + self.assertIsNone(found["state"]) + self.assertIn("The inventory holds no server of this cluster", found["problem"]) + + def test_input_that_is_not_a_list_is_an_error(self) -> None: + with self.assertRaises(guard.AnsibleFilterError): + guard.cluster_guard_state("a") + with self.assertRaises(guard.AnsibleFilterError): + guard.cluster_guard_state(["a"]) + + def test_the_filter_is_registered(self) -> None: + self.assertIs(guard.FilterModule().filters()["cluster_guard_state"], guard.cluster_guard_state) + + +IP_ADDR = [ + "1: lo inet 127.0.0.1/8 scope host lo\\ valid_lft forever preferred_lft forever", + "2: eth0 inet 192.168.5.1/24 metric 200 brd 192.168.5.255 scope global dynamic eth0\\ valid_lft 85000sec", + "3: col0 inet 192.0.2.44/24 metric 300 brd 192.0.2.255 scope global dynamic col0\\ valid_lft 85000sec", + "4: docker0 inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0\\ valid_lft forever", +] + + +class AddressesInNetworkTest(unittest.TestCase): + def test_an_address_in_the_network_is_found(self) -> None: + self.assertEqual(guard.addresses_in_network(IP_ADDR, "192.0.2.0/24"), ["192.0.2.44"]) + + def test_no_address_in_the_network_is_an_empty_list(self) -> None: + self.assertEqual(guard.addresses_in_network(IP_ADDR, "198.51.100.0/24"), []) + + def test_the_network_is_matched_by_prefix_not_by_text(self) -> None: + self.assertEqual(guard.addresses_in_network(IP_ADDR, "192.0.0.0/16"), ["192.0.2.44"]) + self.assertEqual(guard.addresses_in_network(["5: x inet 192.0.20.4/24 scope global x"], "192.0.2.0/24"), []) + + def test_lines_without_an_address_are_skipped(self) -> None: + self.assertEqual(guard.addresses_in_network(["", "garbage", "6: tun inet"], "192.0.2.0/24"), []) + + def test_an_invalid_network_is_an_error(self) -> None: + for network in ("", "192.0.2.0/33", "not-a-network", "2001:db8::/32"): + with self.subTest(network=network), self.assertRaises(guard.AnsibleFilterError): + guard.addresses_in_network(IP_ADDR, network) + + def test_a_string_instead_of_lines_is_an_error(self) -> None: + with self.assertRaises(guard.AnsibleFilterError): + guard.addresses_in_network("\\n".join(IP_ADDR), "192.0.2.0/24") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_cluster_network.py b/tests/unit/test_cluster_network.py new file mode 100644 index 0000000..bed46aa --- /dev/null +++ b/tests/unit/test_cluster_network.py @@ -0,0 +1,120 @@ +"""Tests for the cluster role's node network mode: the Compose project docker compose renders from docker-compose.yml and, with github_runner_cluster_network_mode host, docker-compose.host-network.yml, and the role's derived list of Compose files (tests/cluster_network/compose_files.yml, run with ansible-playbook).""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[2] +FILES = ROOT / "roles" / "github_runner_cluster" / "files" +BASE = FILES / "docker-compose.yml" +HOST_NETWORK = FILES / "docker-compose.host-network.yml" +COMPOSE_FILES_PLAYBOOK = ROOT / "tests" / "cluster_network" / "compose_files.yml" +API_PORT = 6443 +# What .env supplies; docker-compose.yml refuses to render without the required ones. +ENV = {"K3S_TOKEN": "example-token", "K3S_TLS_SAN_LIST": "node1.example.internal", "K3S_NODE_NAME": "node1", "K3S_AGENT_NODE_NAME": "node2", "K3S_VERSION": "v1.34.1-k3s1", "TAILSCALE_VERSION": "v1.102.4"} + + +def has_compose() -> bool: + if shutil.which("docker") is None: + return False + return subprocess.run(["docker", "compose", "version"], capture_output=True, check=False).returncode == 0 + + +# One project directory for every render, so that bind-mount paths, which Compose resolves against it, compare equal between renders. +PROJECT_DIR = Path(tempfile.gettempdir()) / "github-runner-cluster-network-test" + + +def render(*files: Path) -> dict[str, dict[str, Any]]: + """Render the Compose project from the given files, in merge order, with both profiles, and return its services.""" + args = ["docker", "compose", "--project-directory", str(PROJECT_DIR), "--project-name", "example"] + for path in files: + args += ["-f", str(path)] + args += ["--profile", "server", "--profile", "agent", "config", "--format", "json"] + env = {key: value for key, value in os.environ.items() if not key.startswith(("DOCKER_", "COMPOSE_"))} | ENV + result = subprocess.run(args, capture_output=True, text=True, check=False, env=env) + if result.returncode != 0: + raise AssertionError(f"docker compose config failed: {result.stderr}") + services: dict[str, dict[str, Any]] = json.loads(result.stdout)["services"] + return services + + +def published(service: dict[str, Any]) -> set[int]: + return {int(port["published"]) for port in service.get("ports", [])} + + +@unittest.skipUnless(has_compose(), "docker compose is not installed") +class RenderedComposeTest(unittest.TestCase): + def test_bridge_publishes_the_api_server_and_sets_no_network_mode(self) -> None: + services = render(BASE) + self.assertEqual(published(services["k3s"]), {API_PORT}) + for name in ("k3s", "k3s-agent"): + with self.subTest(service=name): + self.assertNotIn("network_mode", services[name]) + + def test_host_puts_both_node_services_on_the_host_network_and_publishes_nothing(self) -> None: + services = render(BASE, HOST_NETWORK) + for name in ("k3s", "k3s-agent"): + with self.subTest(service=name): + self.assertEqual(services[name]["network_mode"], "host") + self.assertEqual(published(services[name]), set()) + self.assertNotIn("networks", services[name]) + + def test_host_changes_nothing_else_about_the_node_services(self) -> None: + bridge, host = render(BASE), render(BASE, HOST_NETWORK) + for name in ("k3s", "k3s-agent"): + with self.subTest(service=name): + for key in ("hostname", "environment", "volumes", "entrypoint", "privileged", "container_name"): + self.assertEqual(host[name].get(key), bridge[name].get(key), key) + self.assertEqual(host["docker-prune"], bridge["docker-prune"]) + + def test_host_empties_a_port_list_an_earlier_override_published(self) -> None: + # As the headscale_in_cluster provider's override does on its bootstrap server; the host network file comes after it in the list. + with tempfile.TemporaryDirectory() as work: + mesh = Path(work) / "docker-compose.mesh.yml" + mesh.write_text('services:\n k3s:\n ports:\n - "8080:8080"\n') + self.assertEqual(published(render(BASE, mesh)["k3s"]), {API_PORT, 8080}) + self.assertEqual(published(render(BASE, mesh, HOST_NETWORK)["k3s"]), set()) + + +@unittest.skipIf(shutil.which(os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook")) is None, "ansible-playbook is not installed") +class ComposeFileListTest(unittest.TestCase): + def file_list(self, **role_vars: Any) -> list[str]: + with tempfile.TemporaryDirectory() as work: + output = Path(work) / "files.json" + extra = Path(work) / "extra.json" + extra.write_text(json.dumps({"compose_files_output": str(output), **role_vars})) + env = {**os.environ, "ANSIBLE_LOCALHOST_WARNING": "false", "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false"} + subprocess.run([os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook"), str(COMPOSE_FILES_PLAYBOOK), "-e", f"@{extra}"], check=True, env=env, cwd=work, capture_output=True, text=True) + files: list[str] = json.loads(output.read_text()) + return files + + def test_bridge_is_the_default_and_adds_no_file(self) -> None: + self.assertEqual(self.file_list(), ["docker-compose.yml"]) + self.assertEqual(self.file_list(github_runner_cluster_network_mode="bridge"), ["docker-compose.yml"]) + + def test_host_adds_the_host_network_file(self) -> None: + self.assertEqual(self.file_list(github_runner_cluster_network_mode="host"), ["docker-compose.yml", "docker-compose.host-network.yml"]) + + def test_the_host_network_file_comes_after_every_other_override(self) -> None: + files = self.file_list( + github_runner_cluster_network_mode="host", + github_runner_cluster_mesh="headscale_in_cluster", + github_runner_cluster_effective_bootstrap=True, + github_runner_cluster_compose_files=["compose.extra.yml"], + ) + self.assertEqual(files, ["docker-compose.yml", "docker-compose.mesh.yml", "compose.extra.yml", "docker-compose.host-network.yml"]) + + def test_the_role_copies_the_host_network_file_to_the_host(self) -> None: + tasks = (ROOT / "roles" / "github_runner_cluster" / "tasks" / "main.yml").read_text() + self.assertIn("{src: docker-compose.host-network.yml, mode: \"0644\"}", tasks) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_host_network_role.py b/tests/unit/test_host_network_role.py new file mode 100644 index 0000000..7b6f99a --- /dev/null +++ b/tests/unit/test_host_network_role.py @@ -0,0 +1,308 @@ +"""Tests for the cluster role's tasks/host_network.yml, run with ansible-playbook (tests/host_network/play.yml) against a stand-in docker command that records what it is asked and answers from files the test writes: the network mode must be one the role knows; with the host network the Docker host must have an address on the LAN and no other Tailscale device; and moving a running node between networks is refused while the cluster needs it or cannot be read, or when another node moves in the same run. Nothing the tasks run may change anything. Also what tasks/main.yml decides about the network, read from the file.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +PLAY = ROOT / "tests" / "host_network" / "play.yml" +ANSIBLE_PLAYBOOK = os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook") +LAN = "192.0.2.0/24" +ON_LAN = "3: col0 inet 192.0.2.44/24 brd 192.0.2.255 scope global col0\\ valid_lft 85000sec" +OFF_LAN = "2: eth0 inet 192.168.5.1/24 brd 192.168.5.255 scope global eth0\\ valid_lft 85000sec" + +# The stand-in docker appends its arguments and DOCKER_CONTEXT to calls.log and answers from files in the work directory: docker ps lists container-state (empty: no such container), docker run prints addresses, a "--" line and links, and docker exec answers kubectl: /readyz/etcd from etcd- (no file: the read fails), the node list from nodes.json and the pods from pods.json. A daemon-down file makes every call fail, kubectl-nodes-fails (or kubectl-nodes-fails-) makes the node list fail, and hang- makes docker exec on that context sleep for the seconds it holds. +STUBS = { + "docker": """\ + #!/bin/sh + echo "docker ${DOCKER_CONTEXT:--} $*" >> "$STUB_DIR/calls.log" + [ -e "$STUB_DIR/daemon-down" ] && { echo "Cannot connect to the Docker daemon" >&2; exit 1; } + case "$1" in + ps) cat "$STUB_DIR/container-state" ;; + run) cat "$STUB_DIR/addresses"; echo --; cat "$STUB_DIR/links" ;; + exec) + [ -e "$STUB_DIR/hang-${DOCKER_CONTEXT:--}" ] && sleep "$(cat "$STUB_DIR/hang-${DOCKER_CONTEXT:--}")" + case "$5" in + --request-timeout=*) [ -e "$STUB_DIR/etcd-$2" ] || { echo "Error from server" >&2; exit 1; }; cat "$STUB_DIR/etcd-$2" ;; + nodes) [ -e "$STUB_DIR/kubectl-nodes-fails" ] || [ -e "$STUB_DIR/kubectl-nodes-fails-$2" ] && { echo "The connection to the server 127.0.0.1:6443 was refused" >&2; exit 1; }; cat "$STUB_DIR/nodes.json" ;; + pods) cat "$STUB_DIR/pods.json" ;; + esac ;; + esac + """, +} +LINKS = "1: lo: mtu 65536 qdisc noqueue state UNKNOWN\\ link/loopback 00:00:00:00:00:00\n2: eth0: mtu 1500 qdisc fq_codel state UP\\ link/ether 02:00:00:00:00:01" +TAILSCALE_LINK = "5: tailscale0: mtu 1280 qdisc fq_codel state UNKNOWN\\ link/none" +TOPOLOGY = {"hosts": {name: {"role": "server", "bootstrap": name == "node1", "node_name": name} for name in ("node1", "node2", "node3")}} + + +def node(name: str, ready: bool = True, labelled: bool = True) -> dict[str, Any]: + labels = {"node-role.kubernetes.io/etcd": "true"} if labelled else {} + return {"metadata": {"name": name, "labels": labels}, "status": {"conditions": [{"type": "Ready", "status": "True" if ready else "Unknown"}]}} + + +RUNNER = {"metadata": {"name": "runner-1", "namespace": "runners", "labels": {"actions-ephemeral-runner": "True"}}, "spec": {"nodeName": "node1"}, "status": {"phase": "Running"}} +HEALTHY = [node("node1"), node("node2"), node("node3")] + + +def failed_task(stdout: str) -> str | None: + """Return the name of the task whose header last precedes a "fatal:" line in ansible-playbook's output, or None when nothing failed.""" + task = None + for line in stdout.splitlines(): + if line.startswith("TASK [") and line.rstrip("*").rstrip().endswith("]"): + task = line[len("TASK [") : line.rstrip("*").rstrip().rindex("]")] + elif line.startswith("fatal: "): + return task + return None + + +@unittest.skipIf(shutil.which(ANSIBLE_PLAYBOOK) is None, "ansible-playbook is not installed") +class HostNetworkRoleTest(unittest.TestCase): + def setUp(self) -> None: + self.work = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.work) + self.bin = self.work / "bin" + self.bin.mkdir() + for name, script in STUBS.items(): + path = self.bin / name + path.write_text(textwrap.dedent(script)) + path.chmod(0o755) + inventory = {name: {"github_runner_cluster_container_name": f"{name}-k3s"} for name in ("node1", "node2", "node3")} + inventory["node2"]["github_runner_cluster_docker_context"] = "ctx2" + inventory["node3"]["github_runner_cluster_docker_context"] = "ctx3" + (self.work / "inventory.json").write_text(json.dumps({"all": {"hosts": inventory}})) + self.state(addresses=[OFF_LAN, ON_LAN], container="running github-runner_default", nodes=HEALTHY, pods=[]) + self.etcd("node2-k3s", "node3-k3s") + + def state(self, addresses: list[str], container: str, nodes: list[dict[str, Any]], pods: list[dict[str, Any]], links: str = LINKS) -> None: + (self.work / "addresses").write_text("\n".join(addresses) + "\n") + (self.work / "links").write_text(links + "\n") + (self.work / "container-state").write_text(container + "\n" if container else "") + (self.work / "nodes.json").write_text(json.dumps({"items": nodes})) + (self.work / "pods.json").write_text(json.dumps({"items": pods})) + + def etcd(self, *healthy: str) -> None: + for old in self.work.glob("etcd-*"): + old.unlink() + for container in healthy: + (self.work / f"etcd-{container}").write_text("ok") + + def run_tasks(self, *args: str, limit: str = "node1", timeout: int | None = None, **role_vars: Any) -> subprocess.CompletedProcess[str]: + extra = { + "github_runner_cluster_network_mode": "host", + "github_runner_cluster_lan_cidr": LAN, + "github_runner_cluster_effective_node_role": "server", + "github_runner_cluster_effective_node_name": "node1", + "github_runner_cluster_topology": TOPOLOGY, + **role_vars, + } + extra_file = self.work / "extra.json" + extra_file.write_text(json.dumps(extra)) + env = { + **os.environ, + "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}", + "STUB_DIR": str(self.work), + "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), + "ANSIBLE_LOCALHOST_WARNING": "false", + "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", + "ANSIBLE_RETRY_FILES_ENABLED": "false", + } + command = [ANSIBLE_PLAYBOOK, "-i", str(self.work / "inventory.json"), str(PLAY), "--limit", limit, "-e", f"@{extra_file}", *args] + return subprocess.run(command, capture_output=True, text=True, check=False, env=env, cwd=self.work, timeout=timeout) + + def calls(self) -> list[str]: + log = self.work / "calls.log" + return log.read_text().splitlines() if log.exists() else [] + + def test_moving_a_healthy_node_to_the_host_network_passes_after_asking_the_cluster(self) -> None: + result = self.run_tasks() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + calls = self.calls() + self.assertIn("docker - run --rm --network host busybox:1.37 sh -c ip -4 -o addr show && echo -- && ip -o link show", calls) + self.assertTrue("docker - exec github-runner-k3s kubectl get nodes --request-timeout=30s -o json" in calls, calls) + self.assertIn("docker - exec github-runner-k3s kubectl get pods --all-namespaces --request-timeout=30s --field-selector spec.nodeName=node1 -o json", calls) + self.assertIn("docker ctx2 exec node2-k3s kubectl get --request-timeout=30s --raw /readyz/etcd", calls) + self.assertIn("docker ctx3 exec node3-k3s kubectl get --request-timeout=30s --raw /readyz/etcd", calls) + + def test_the_cluster_reads_run_under_check(self) -> None: + self.state(addresses=[ON_LAN], container="running github-runner_default", nodes=[node("node1"), node("node2"), node("node3", ready=False)], pods=[]) + result = self.run_tasks("--check") + self.assertNotEqual(result.returncode, 0) + self.assertIn("node3 (not Ready)", result.stdout) + + def test_a_healthy_cluster_passes_under_check(self) -> None: + result = self.run_tasks("--check") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_a_server_whose_etcd_is_not_healthy_refuses_the_move(self) -> None: + self.etcd("node2-k3s") + result = self.run_tasks() + self.assertNotEqual(result.returncode, 0) + self.assertIn("node3 (etcd not reported healthy)", result.stdout) + + def test_servers_without_role_labels_refuse_the_move(self) -> None: + self.state(addresses=[ON_LAN], container="running github-runner_default", nodes=[node("node1", labelled=False), node("node2", labelled=False), node("node3", ready=False, labelled=False)], pods=[]) + result = self.run_tasks() + self.assertNotEqual(result.returncode, 0) + self.assertIn("does not label node1, node2, node3 as server nodes", result.stdout) + + def test_a_hung_node_read_on_this_host_is_cut_off_and_another_server_answers(self) -> None: + (self.work / "hang--").write_text("60") + result = self.run_tasks(github_runner_cluster_cluster_read_timeout=2, timeout=60) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("docker ctx2 exec node2-k3s kubectl get nodes --request-timeout=2s -o json", self.calls()) + + def test_a_hung_etcd_read_is_cut_off_and_counts_as_unhealthy(self) -> None: + (self.work / "hang-ctx3").write_text("60") + result = self.run_tasks(github_runner_cluster_cluster_read_timeout=2, timeout=60) + self.assertNotEqual(result.returncode, 0) + self.assertIn("node3 (etcd not reported healthy)", result.stdout) + + def test_a_cluster_no_server_can_be_read_through_refuses_and_says_why(self) -> None: + (self.work / "kubectl-nodes-fails").write_text("") + result = self.run_tasks() + self.assertNotEqual(result.returncode, 0) + self.assertEqual(failed_task(result.stdout), "Fail if no server of the inventory could be read") + self.assertIn("node1 could not be read (The connection to the server 127.0.0.1:6443 was refused)", result.stdout) + self.assertIn("node3 could not be read", result.stdout) + + def test_the_cluster_is_read_through_another_server_when_this_one_cannot_list_it(self) -> None: + (self.work / "kubectl-nodes-fails-github-runner-k3s").write_text("") + result = self.run_tasks() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("docker ctx2 exec node2-k3s kubectl get nodes --request-timeout=30s -o json", self.calls()) + + def test_an_agent_in_a_group_with_no_bootstrap_host_reads_the_servers(self) -> None: + topology = {"hosts": {"node1": {"role": "server", "bootstrap": False, "node_name": "node1"}, "node2": {"role": "agent", "bootstrap": False, "node_name": "node2"}}} + self.state(addresses=[ON_LAN], container="running github-runner_default", nodes=[node("node1"), node("node2", labelled=False)], pods=[]) + result = self.run_tasks(limit="node2", github_runner_cluster_effective_node_role="agent", github_runner_cluster_effective_node_name="node2", github_runner_cluster_topology=topology) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("docker - exec node1-k3s kubectl get nodes --request-timeout=30s -o json", self.calls()) + + def test_an_agent_whose_servers_are_outside_the_inventory_says_why_it_cannot_move(self) -> None: + topology = {"hosts": {"node2": {"role": "agent", "bootstrap": False, "node_name": "node2"}}} + result = self.run_tasks(limit="node2", github_runner_cluster_effective_node_role="agent", github_runner_cluster_effective_node_name="node2", github_runner_cluster_topology=topology) + self.assertNotEqual(result.returncode, 0) + self.assertIn("The inventory holds no server of this cluster", result.stdout) + + def test_a_docker_daemon_that_cannot_be_reached_fails_the_run(self) -> None: + (self.work / "daemon-down").write_text("") + result = self.run_tasks(github_runner_cluster_network_mode="bridge", github_runner_cluster_lan_cidr="") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(failed_task(result.stdout), "Read whether the node's container runs, and on which network") + + def test_an_unknown_network_mode_fails_before_running_anything(self) -> None: + result = self.run_tasks(github_runner_cluster_network_mode="macvlan") + self.assertNotEqual(result.returncode, 0) + self.assertIn("it must be bridge", result.stdout) + self.assertEqual(self.calls(), []) + + def test_two_nodes_moving_in_one_run_are_refused(self) -> None: + result = self.run_tasks(limit="node1,node2") + self.assertNotEqual(result.returncode, 0) + self.assertIn("This run would move node1, node2 to another network at once", result.stdout) + self.assertFalse(any(" kubectl get " in call for call in self.calls())) + + def test_another_tailscale_device_on_the_host_network_refuses_host_mode(self) -> None: + for container in ("running github-runner_default", "", "exited host"): + with self.subTest(container=container): + self.state(addresses=[ON_LAN], container=container, nodes=HEALTHY, pods=[], links=LINKS + "\n" + TAILSCALE_LINK) + result = self.run_tasks() + self.assertNotEqual(result.returncode, 0) + self.assertIn("already has a Tailscale device on its network (tailscale0)", result.stdout) + + def test_the_node_s_own_tailscale_device_is_expected_once_it_runs_on_the_host_network(self) -> None: + self.state(addresses=[ON_LAN], container="running host", nodes=HEALTHY, pods=[], links=LINKS + "\n" + TAILSCALE_LINK) + result = self.run_tasks() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_no_address_on_the_lan_fails_and_names_the_addresses_found(self) -> None: + self.state(addresses=[OFF_LAN], container="running github-runner_default", nodes=HEALTHY, pods=[]) + result = self.run_tasks() + self.assertNotEqual(result.returncode, 0) + self.assertIn("has no address in 192.0.2.0/24", result.stdout) + self.assertIn("192.168.5.1/24", result.stdout) + self.assertFalse(any(" kubectl get " in call for call in self.calls())) + + def test_the_host_network_without_a_lan_fails_before_running_anything(self) -> None: + result = self.run_tasks(github_runner_cluster_lan_cidr="") + self.assertNotEqual(result.returncode, 0) + self.assertIn("github_runner_cluster_lan_cidr is empty", result.stdout) + self.assertEqual(self.calls(), []) + + def test_a_server_not_ready_refuses_the_move(self) -> None: + self.state(addresses=[ON_LAN], container="running github-runner_default", nodes=[node("node1"), node("node2"), node("node3", ready=False)], pods=[]) + result = self.run_tasks() + self.assertNotEqual(result.returncode, 0) + self.assertIn("Not moving node1 to the host network", result.stdout) + self.assertIn("node3 (not Ready)", result.stdout) + + def test_a_runner_pod_on_the_node_refuses_the_move(self) -> None: + self.state(addresses=[ON_LAN], container="running github-runner_default", nodes=HEALTHY, pods=[RUNNER]) + result = self.run_tasks() + self.assertNotEqual(result.returncode, 0) + self.assertIn("runners/runner-1", result.stdout) + + def test_an_agent_reads_the_cluster_through_the_bootstrap_server_and_no_etcd(self) -> None: + self.state(addresses=[ON_LAN], container="running github-runner_default", nodes=[*HEALTHY, node("node4", labelled=False)], pods=[]) + result = self.run_tasks(limit="node2", github_runner_cluster_effective_node_role="agent", github_runner_cluster_effective_node_name="node4") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertFalse(any("/readyz/etcd" in call for call in self.calls())) + self.assertIn("docker - exec node1-k3s kubectl get nodes --request-timeout=30s -o json", self.calls()) + self.assertIn("docker - ps --all --filter name=^/github-runner-k3s-agent$ --format {{.State}} {{.Networks}}", self.calls()) + + def test_a_node_already_on_the_host_network_is_not_checked_against_the_cluster(self) -> None: + self.state(addresses=[ON_LAN], container="running host", nodes=[node("node1"), node("node2", ready=False), node("node3", ready=False)], pods=[RUNNER]) + result = self.run_tasks() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertFalse(any(" kubectl get " in call for call in self.calls())) + + def test_a_node_that_is_not_running_or_missing_is_not_checked_against_the_cluster(self) -> None: + for container in ("exited github-runner_default", ""): + with self.subTest(container=container): + self.state(addresses=[ON_LAN], container=container, nodes=[node("node1"), node("node2", ready=False)], pods=[RUNNER]) + (self.work / "calls.log").unlink(missing_ok=True) + result = self.run_tasks() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertFalse(any(" kubectl get " in call for call in self.calls())) + + def test_moving_back_to_bridge_is_guarded_too_and_needs_no_lan(self) -> None: + self.state(addresses=[OFF_LAN], container="running host", nodes=HEALTHY, pods=[RUNNER]) + result = self.run_tasks(github_runner_cluster_network_mode="bridge", github_runner_cluster_lan_cidr="") + self.assertNotEqual(result.returncode, 0) + self.assertIn("Not moving node1 to the bridge network", result.stdout) + self.assertFalse(any(" run " in call for call in self.calls())) + + def test_bridge_staying_bridge_reads_only_the_container(self) -> None: + result = self.run_tasks(github_runner_cluster_network_mode="bridge", github_runner_cluster_lan_cidr="") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual([call.split()[2] for call in self.calls()], ["ps"]) + + +class MainTasksTest(unittest.TestCase): + """What tasks/main.yml decides about the network mode, read from the file, since it runs only against a real Docker host.""" + + def test_the_loopback_api_wait_runs_only_on_the_host_network(self) -> None: + tasks = yaml.safe_load((ROOT / "roles" / "github_runner_cluster" / "tasks" / "main.yml").read_text()) + wait = next(task for task in tasks if "ansible.builtin.wait_for" in task) + self.assertEqual(wait["ansible.builtin.wait_for"]["host"], "127.0.0.1") + self.assertIn("github_runner_cluster_network_mode == 'host'", wait["when"]) + + def test_the_network_checks_run_before_anything_changes(self) -> None: + tasks = yaml.safe_load((ROOT / "roles" / "github_runner_cluster" / "tasks" / "main.yml").read_text()) + includes = [task.get("ansible.builtin.include_tasks") for task in tasks] + first_change = next(index for index, task in enumerate(tasks) if "ansible.builtin.file" in task or "ansible.builtin.copy" in task or "block" in task) + self.assertLess(includes.index("host_network.yml"), first_change) + + +if __name__ == "__main__": + unittest.main()