feat(check): rename the status to Vapi Simulations; one target, one status

.github/workflows/vapi-checks.yml

@@ -16,13 +16,13 @@ on:
       check:
         description: >-
           Check name from vapi-checks.yml. Leave blank to run every check and
-          update the Vapi Evals status on the dispatched commit.
+          update the Vapi Simulations status on the dispatched commit.
         required: false
         type: string
 
 permissions:
   contents: read
-  # Only to post the `Vapi Evals` commit statuses that link to each run.
+  # Only to post the `Vapi Simulations` commit statuses that link to each run.
   statuses: write
 
 concurrency:

README.md

@@ -20,7 +20,7 @@ come back here to manage what you build.
   per org.
 - **Tests on every pull request.** Simulation suites run against the branch's
   own files, with tools mocked and nothing deployed, and report a
-  `Vapi Evals` status that links to the run.
+  `Vapi Simulations` status that links to the run.
 - **A field guide to Vapi.** [`docs/learnings/`](docs/learnings/README.md)
   collects hard-won gotchas and recipes for assistants, squads, transfers,
   voicemail detection, latency and more.

docs/guides/pr-checks.md

@@ -106,32 +106,38 @@ PR push. It asks for `statuses: write` only to post the direct links.
 
 ## 6. What a PR shows
 
-- `Vapi Evals`, plus `Vapi Evals / <check> / <target>` per target. **Details**
-  opens the run in Vapi.
+- `Vapi Simulations`, the overall result. When one target ran, **Details**
+  opens its run in Vapi. When several did, each also gets
+  `Vapi Simulations / <check> / <target>`, whose **Details** opens that run,
+  and the overall status links to the workflow run.
 - A job summary: per-target result, failing judges with expected vs actual,
   and any "unmocked tool called" notices. No PR comments.
 - PRs that touch neither a check's org, its state, `vapi-checks.yml`,
   `promotion.yml`, the engine (`src/**`, `package*.json`), nor the check's
-  own `paths` skip it, and `Vapi Evals` posts success.
+  own `paths` skip it, and `Vapi Simulations` posts success.
 - A newer push cancels the older run.
 - Separately, the **Validate resources** check (in `ci.yml`) runs
   `npm run validate` on every org, including resources no check targets.
   It's offline and runs whether or not PR checks are turned on.
 
 ## 7. Make it required (after a burn-in)
 
-Require the **commit status `Vapi Evals`** in branch protection — not the
+Require the **commit status `Vapi Simulations`** in branch protection — not the
 `vapi-checks` job (fork dry runs succeed) and not the per-target statuses
 (PRs that don't touch a check never get them).
 
 - **Fork PRs** run a dry run without secrets and can't post statuses (the
-  token is read-only), so a required `Vapi Evals` blocks them.
+  token is read-only), so a required `Vapi Simulations` blocks them.
 - **Dependabot PRs** that change `package*.json` post `error`.
 - **To unblock either**, a maintainer runs Actions → Vapi checks → Run
   workflow on the PR's branch with `check` blank (push a fork's branch into
   the repository first). A later PR event on the same commit resets the
   status, so dispatch again after that. Running one named check by hand
-  never changes `Vapi Evals`.
+  never changes `Vapi Simulations`.
+
+This status was called `Vapi Evals` before 2026-10-13. If you already
+require `Vapi Evals`, require `Vapi Simulations` instead after you merge in
+the update, or every pull request waits for a status that no longer posts.
 
 ## Gate promotion on a check (optional)
 

improvements.md

@@ -1850,7 +1850,8 @@ None needed once the fix below lands.
   and servers dead-ended (`src/check-mocks.ts`); the verdict is the strict
   `simRunVerdict` from #33.
 - `.github/workflows/vapi-checks.yml` (opt-in via `VAPI_CHECKS_ENABLED`) runs
-  affected checks on each PR and posts `Vapi Evals` commit statuses that link
+  affected checks on each PR and posts `Vapi Simulations` commit statuses (called `Vapi Evals` until
+  2026-10-13) that link
   straight to the run; forks and Dependabot get a keyless dry run.
 - Docs: README "PR Checks", `docs/learnings/simulations.md` "Inline PR
   Checks".

src/check-cmd.ts

@@ -2,7 +2,7 @@
 //
 // Builds each check target's inline simulation-run payload from the files
 // on disk (check-payload.ts, check-mocks.ts) and, unless --dry-run, runs it
-// in the check's run org and reports a strict verdict — as `Vapi Evals`
+// in the check's run org and reports a strict verdict — as `Vapi Simulations`
 // commit statuses when the PR workflow's GitHub env is set, and always as
 // a markdown report (the job summary in CI).
 //
@@ -42,7 +42,7 @@ const USAGE = [
   "",
   "Options:",
   `  <check>                  A check name from ${CHECKS_CONFIG_FILE}`,
-  "  --all                    Every check (also posts the aggregate Vapi Evals status)",
+  "  --all                    Every check (also posts the aggregate Vapi Simulations status)",
   "  --dry-run                Build the payloads offline; nothing is sent",
   "  --print-payload [dir]    Write each payload as JSON (default: tmp/check-payloads)",
   "  --changed-since <ref>    Only checks affected by changes since the merge base with <ref>",
@@ -297,6 +297,10 @@ async function liveRun(
   const post = async (status: CommitStatus) => {
     if (statusEnv) await commitStatusPost(statusEnv, status);
   };
+  // With one target in an --all run, the aggregate speaks for it: pending here
+  // with the run link, and the verdict (also linking the run) when the run
+  // ends. A per-target status would only repeat it.
+  const single = parsed.all && jobs.length === 1;
   try {
     console.log(
       `\n🧪 Running ${plural(jobs.filter((job) => job.result.body).length, "target")}…`,
@@ -308,21 +312,25 @@ async function liveRun(
       signal: controller.signal,
       onRunCreated: (job, url) =>
         post({
-          context: targetContext(job.check.name, checkTargetLabel(job.target)),
+          context: single
+            ? AGGREGATE_CONTEXT
+            : targetContext(job.check.name, checkTargetLabel(job.target)),
           state: "pending",
           description: "Simulations running",
           targetUrl: url,
         }),
-      onResult: (result) =>
-        post({
+      onResult: async (result) => {
+        if (single) return;
+        await post({
           context: targetContext(
             result.job.check.name,
             checkTargetLabel(result.job.target),
           ),
           state: outcomeState(result.outcome),
           description: result.reason,
           targetUrl: result.url ?? statusEnv?.runUrl,
-        }),
+        });
+      },
     });
   } finally {
     process.off("SIGINT", abort);

src/check-report.ts

@@ -68,7 +68,7 @@ function detailLines(result: CheckTargetResult): string[] {
 
 export function checkReportMarkdown(input: CheckReportInput): string {
   const lines = [
-    `## Vapi Evals${input.dryRun ? " (dry run: payloads built, nothing sent)" : ""}`,
+    `## Vapi Simulations${input.dryRun ? " (dry run: payloads built, nothing sent)" : ""}`,
     "",
   ];
   if (input.results.length > 0) {

src/check-status.ts

@@ -1,13 +1,16 @@
 // GitHub commit statuses for PR checks. Plain fetch, no dependency.
 //
-// Per check and target: `Vapi Evals / <check> / <target>`, pending with the
-// run link as soon as the run exists, then the verdict. The aggregate
-// `Vapi Evals` (a stable, documented name) is what branch protection
-// requires: per-target statuses only exist on PRs that touch a check.
+// The aggregate `Vapi Simulations` (a stable, documented name) is what branch
+// protection requires: per-target statuses only exist on PRs that touch a
+// check. With several targets, each also gets `Vapi Simulations / <check> /
+// <target>`, pending with the run link as soon as the run exists, then the
+// verdict, so every run is one click away. With one, the aggregate carries the
+// pending state and links to the run itself, and no per-target status is
+// posted: it would only repeat the aggregate.
 
 import { userAgentGet } from "./user-agent.ts";
 
-export const AGGREGATE_CONTEXT = "Vapi Evals";
+export const AGGREGATE_CONTEXT = "Vapi Simulations";
 
 export type CommitState = "pending" | "success" | "failure" | "error";
 

tests/check-cmd.test.ts

@@ -279,7 +279,7 @@ function liveCopy(stub: LiveStub): string {
   return root;
 }
 
-test("a live --all run posts pending then success per target, the aggregate, the job summary and JSON", async () => {
+test("with one target, a live --all run posts only the aggregate (pending, then success, both linking the run), plus the job summary and JSON", async () => {
   await withLiveStub(async (stub) => {
     const root = liveCopy(stub);
     try {
@@ -306,17 +306,8 @@ test("a live --all run posts pending then success per target, the aggregate, the
         {
           code: 0,
           statuses: [
-            [
-              "Vapi Evals / core / squads/dental",
-              "pending",
-              "https://dashboard.vapi.ai/run-1",
-            ],
-            [
-              "Vapi Evals / core / squads/dental",
-              "success",
-              "https://dashboard.vapi.ai/run-1",
-            ],
-            ["Vapi Evals", "success", "https://dashboard.vapi.ai/run-1"],
+            ["Vapi Simulations", "pending", "https://dashboard.vapi.ai/run-1"],
+            ["Vapi Simulations", "success", "https://dashboard.vapi.ai/run-1"],
           ],
           summary: true,
           json: ["passed"],
@@ -328,6 +319,43 @@ test("a live --all run posts pending then success per target, the aggregate, the
   });
 });
 
+test("with several targets, each gets its own status linking its run, and the aggregate links the workflow run", async () => {
+  await withLiveStub(async (stub) => {
+    const root = liveCopy(stub);
+    try {
+      writeFileSync(
+        join(root, "vapi-checks.yml"),
+        "version: 1\nchecks:\n  core:\n    org: parity\n    targets: [squads/dental]\n    suites: [core]\n  again:\n    org: parity\n    targets: [squads/dental]\n    suites: [core]\n",
+      );
+      await withGitHubEnv(stub, join(root, "summary.md"), () =>
+        run(["--all"], root),
+      );
+      const contexts = stub.statuses.map((s) => `${s.context} ${s.state}`);
+      const aggregate = stub.statuses.at(-1);
+      assert.deepEqual(
+        {
+          perTarget: contexts.filter((c) => c.includes(" / ")).sort(),
+          aggregate: [aggregate?.context, aggregate?.state],
+          aggregateLinksWorkflow:
+            aggregate?.target_url?.includes("/actions/runs/"),
+        },
+        {
+          perTarget: [
+            "Vapi Simulations / again / squads/dental pending",
+            "Vapi Simulations / again / squads/dental success",
+            "Vapi Simulations / core / squads/dental pending",
+            "Vapi Simulations / core / squads/dental success",
+          ],
+          aggregate: ["Vapi Simulations", "success"],
+          aggregateLinksWorkflow: true,
+        },
+      );
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+});
+
 test("a failing live run exits 1 and turns the aggregate red", async () => {
   await withLiveStub(async (stub) => {
     stub.mode.failFirstItem = true;
@@ -341,7 +369,7 @@ test("a failing live run exits 1 and turns the aggregate red", async () => {
         [
           1,
           {
-            context: "Vapi Evals",
+            context: "Vapi Simulations",
             state: "failure",
             description: "1 failed",
             target_url: "https://dashboard.vapi.ai/run-1",
@@ -364,8 +392,8 @@ test("a single named check never posts the aggregate", async () => {
       assert.deepEqual(
         stub.statuses.map((s) => s.context),
         [
-          "Vapi Evals / core / squads/dental",
-          "Vapi Evals / core / squads/dental",
+          "Vapi Simulations / core / squads/dental",
+          "Vapi Simulations / core / squads/dental",
         ],
       );
     } finally {
@@ -404,7 +432,13 @@ test("an unaffected change runs nothing and posts a green aggregate", async () =
         [
           0,
           0,
-          [["Vapi Evals", "success", "No checks affected by this change"]],
+          [
+            [
+              "Vapi Simulations",
+              "success",
+              "No checks affected by this change",
+            ],
+          ],
         ],
       );
     } finally {
@@ -426,7 +460,7 @@ test("a dry run of an affected PR (fork or Dependabot) posts an error aggregate
           stub.vapiPosts,
           stub.statuses.map((s) => [s.context, s.state]),
         ],
-        [0, 0, [["Vapi Evals", "error"]]],
+        [0, 0, [["Vapi Simulations", "error"]]],
       );
     } finally {
       rmSync(root, { recursive: true, force: true });
@@ -447,7 +481,7 @@ test("an invalid vapi-checks.yml still posts an error aggregate", async () => {
           result.code,
           stub.statuses.map((s) => [s.context, s.state, s.description]),
         ],
-        [2, [["Vapi Evals", "error", "vapi-checks.yml is invalid"]]],
+        [2, [["Vapi Simulations", "error", "vapi-checks.yml is invalid"]]],
       );
     } finally {
       rmSync(root, { recursive: true, force: true });

tests/check-report.test.ts

@@ -69,7 +69,7 @@ test("the markdown report has a summary row per target and detail for anything n
       dryRun: false,
     }),
     [
-      "## Vapi Evals",
+      "## Vapi Simulations",
       "",
       "| Check | Target | Result | Simulations | Run |",
       "|---|---|---|---|---|",

tests/check-status.test.ts

@@ -95,7 +95,7 @@ test("statuses post to the head SHA with the run link, and long descriptions are
         "POST /repos/acme/gitops/statuses/abc",
         "Bearer t",
         {
-          context: "Vapi Evals / core / squads/main",
+          context: "Vapi Simulations / core / squads/main",
           state: "failure",
           description: 140,
           target_url: "https://run",
@@ -110,7 +110,7 @@ test("a rejected status (read-only token) only warns", async () => {
     assert.equal(
       await commitStatusPost(
         { token: "t", repo: "acme/gitops", sha: "abc", apiUrl },
-        { context: "Vapi Evals", state: "success", description: "ok" },
+        { context: "Vapi Simulations", state: "success", description: "ok" },
       ),
       false,
     );