diff --git a/.github/workflows/vapi-checks.yml b/.github/workflows/vapi-checks.yml index f89b070..0477d15 100644 --- a/.github/workflows/vapi-checks.yml +++ b/.github/workflows/vapi-checks.yml @@ -16,13 +16,13 @@ on: check: description: >- Check name from vapi-checks.yml. Leave blank to run every check and - update the Vapi Evals status on the dispatched commit. + update the Vapi Simulations status on the dispatched commit. required: false type: string permissions: contents: read - # Only to post the `Vapi Evals` commit statuses that link to each run. + # Only to post the `Vapi Simulations` commit statuses that link to each run. statuses: write concurrency: diff --git a/README.md b/README.md index d5fbfae..6565d7b 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ come back here to manage what you build. per org. - **Tests on every pull request.** Simulation suites run against the branch's own files, with tools mocked and nothing deployed, and report a - `Vapi Evals` status that links to the run. + `Vapi Simulations` status that links to the run. - **A field guide to Vapi.** [`docs/learnings/`](docs/learnings/README.md) collects hard-won gotchas and recipes for assistants, squads, transfers, voicemail detection, latency and more. diff --git a/docs/guides/pr-checks.md b/docs/guides/pr-checks.md index fdfa4ff..58b514f 100644 --- a/docs/guides/pr-checks.md +++ b/docs/guides/pr-checks.md @@ -106,13 +106,15 @@ PR push. It asks for `statuses: write` only to post the direct links. ## 6. What a PR shows -- `Vapi Evals`, plus `Vapi Evals / / ` per target. **Details** - opens the run in Vapi. +- `Vapi Simulations`, the overall result. When one target ran, **Details** + opens its run in Vapi. When several did, each also gets + `Vapi Simulations / / `, whose **Details** opens that run, + and the overall status links to the workflow run. - A job summary: per-target result, failing judges with expected vs actual, and any "unmocked tool called" notices. No PR comments. - PRs that touch neither a check's org, its state, `vapi-checks.yml`, `promotion.yml`, the engine (`src/**`, `package*.json`), nor the check's - own `paths` skip it, and `Vapi Evals` posts success. + own `paths` skip it, and `Vapi Simulations` posts success. - A newer push cancels the older run. - Separately, the **Validate resources** check (in `ci.yml`) runs `npm run validate` on every org, including resources no check targets. @@ -120,18 +122,22 @@ PR push. It asks for `statuses: write` only to post the direct links. ## 7. Make it required (after a burn-in) -Require the **commit status `Vapi Evals`** in branch protection — not the +Require the **commit status `Vapi Simulations`** in branch protection — not the `vapi-checks` job (fork dry runs succeed) and not the per-target statuses (PRs that don't touch a check never get them). - **Fork PRs** run a dry run without secrets and can't post statuses (the - token is read-only), so a required `Vapi Evals` blocks them. + token is read-only), so a required `Vapi Simulations` blocks them. - **Dependabot PRs** that change `package*.json` post `error`. - **To unblock either**, a maintainer runs Actions → Vapi checks → Run workflow on the PR's branch with `check` blank (push a fork's branch into the repository first). A later PR event on the same commit resets the status, so dispatch again after that. Running one named check by hand - never changes `Vapi Evals`. + never changes `Vapi Simulations`. + +This status was called `Vapi Evals` before 2026-10-13. If you already +require `Vapi Evals`, require `Vapi Simulations` instead after you merge in +the update, or every pull request waits for a status that no longer posts. ## Gate promotion on a check (optional) diff --git a/improvements.md b/improvements.md index 43be71a..ef4ca0e 100644 --- a/improvements.md +++ b/improvements.md @@ -1850,7 +1850,8 @@ None needed once the fix below lands. and servers dead-ended (`src/check-mocks.ts`); the verdict is the strict `simRunVerdict` from #33. - `.github/workflows/vapi-checks.yml` (opt-in via `VAPI_CHECKS_ENABLED`) runs - affected checks on each PR and posts `Vapi Evals` commit statuses that link + affected checks on each PR and posts `Vapi Simulations` commit statuses (called `Vapi Evals` until + 2026-10-13) that link straight to the run; forks and Dependabot get a keyless dry run. - Docs: README "PR Checks", `docs/learnings/simulations.md` "Inline PR Checks". diff --git a/src/check-cmd.ts b/src/check-cmd.ts index 5364b08..ddcb137 100644 --- a/src/check-cmd.ts +++ b/src/check-cmd.ts @@ -2,7 +2,7 @@ // // Builds each check target's inline simulation-run payload from the files // on disk (check-payload.ts, check-mocks.ts) and, unless --dry-run, runs it -// in the check's run org and reports a strict verdict — as `Vapi Evals` +// in the check's run org and reports a strict verdict — as `Vapi Simulations` // commit statuses when the PR workflow's GitHub env is set, and always as // a markdown report (the job summary in CI). // @@ -42,7 +42,7 @@ const USAGE = [ "", "Options:", ` A check name from ${CHECKS_CONFIG_FILE}`, - " --all Every check (also posts the aggregate Vapi Evals status)", + " --all Every check (also posts the aggregate Vapi Simulations status)", " --dry-run Build the payloads offline; nothing is sent", " --print-payload [dir] Write each payload as JSON (default: tmp/check-payloads)", " --changed-since Only checks affected by changes since the merge base with ", @@ -297,6 +297,10 @@ async function liveRun( const post = async (status: CommitStatus) => { if (statusEnv) await commitStatusPost(statusEnv, status); }; + // With one target in an --all run, the aggregate speaks for it: pending here + // with the run link, and the verdict (also linking the run) when the run + // ends. A per-target status would only repeat it. + const single = parsed.all && jobs.length === 1; try { console.log( `\n🧪 Running ${plural(jobs.filter((job) => job.result.body).length, "target")}…`, @@ -308,13 +312,16 @@ async function liveRun( signal: controller.signal, onRunCreated: (job, url) => post({ - context: targetContext(job.check.name, checkTargetLabel(job.target)), + context: single + ? AGGREGATE_CONTEXT + : targetContext(job.check.name, checkTargetLabel(job.target)), state: "pending", description: "Simulations running", targetUrl: url, }), - onResult: (result) => - post({ + onResult: async (result) => { + if (single) return; + await post({ context: targetContext( result.job.check.name, checkTargetLabel(result.job.target), @@ -322,7 +329,8 @@ async function liveRun( state: outcomeState(result.outcome), description: result.reason, targetUrl: result.url ?? statusEnv?.runUrl, - }), + }); + }, }); } finally { process.off("SIGINT", abort); diff --git a/src/check-report.ts b/src/check-report.ts index 93c998b..25af8a5 100644 --- a/src/check-report.ts +++ b/src/check-report.ts @@ -68,7 +68,7 @@ function detailLines(result: CheckTargetResult): string[] { export function checkReportMarkdown(input: CheckReportInput): string { const lines = [ - `## Vapi Evals${input.dryRun ? " (dry run: payloads built, nothing sent)" : ""}`, + `## Vapi Simulations${input.dryRun ? " (dry run: payloads built, nothing sent)" : ""}`, "", ]; if (input.results.length > 0) { diff --git a/src/check-status.ts b/src/check-status.ts index 7e4ab34..921c2ad 100644 --- a/src/check-status.ts +++ b/src/check-status.ts @@ -1,13 +1,16 @@ // GitHub commit statuses for PR checks. Plain fetch, no dependency. // -// Per check and target: `Vapi Evals / / `, pending with the -// run link as soon as the run exists, then the verdict. The aggregate -// `Vapi Evals` (a stable, documented name) is what branch protection -// requires: per-target statuses only exist on PRs that touch a check. +// The aggregate `Vapi Simulations` (a stable, documented name) is what branch +// protection requires: per-target statuses only exist on PRs that touch a +// check. With several targets, each also gets `Vapi Simulations / / +// `, pending with the run link as soon as the run exists, then the +// verdict, so every run is one click away. With one, the aggregate carries the +// pending state and links to the run itself, and no per-target status is +// posted: it would only repeat the aggregate. import { userAgentGet } from "./user-agent.ts"; -export const AGGREGATE_CONTEXT = "Vapi Evals"; +export const AGGREGATE_CONTEXT = "Vapi Simulations"; export type CommitState = "pending" | "success" | "failure" | "error"; diff --git a/tests/check-cmd.test.ts b/tests/check-cmd.test.ts index 82c7d5d..1fb8f50 100644 --- a/tests/check-cmd.test.ts +++ b/tests/check-cmd.test.ts @@ -279,7 +279,7 @@ function liveCopy(stub: LiveStub): string { return root; } -test("a live --all run posts pending then success per target, the aggregate, the job summary and JSON", async () => { +test("with one target, a live --all run posts only the aggregate (pending, then success, both linking the run), plus the job summary and JSON", async () => { await withLiveStub(async (stub) => { const root = liveCopy(stub); try { @@ -306,17 +306,8 @@ test("a live --all run posts pending then success per target, the aggregate, the { code: 0, statuses: [ - [ - "Vapi Evals / core / squads/dental", - "pending", - "https://dashboard.vapi.ai/run-1", - ], - [ - "Vapi Evals / core / squads/dental", - "success", - "https://dashboard.vapi.ai/run-1", - ], - ["Vapi Evals", "success", "https://dashboard.vapi.ai/run-1"], + ["Vapi Simulations", "pending", "https://dashboard.vapi.ai/run-1"], + ["Vapi Simulations", "success", "https://dashboard.vapi.ai/run-1"], ], summary: true, json: ["passed"], @@ -328,6 +319,43 @@ test("a live --all run posts pending then success per target, the aggregate, the }); }); +test("with several targets, each gets its own status linking its run, and the aggregate links the workflow run", async () => { + await withLiveStub(async (stub) => { + const root = liveCopy(stub); + try { + writeFileSync( + join(root, "vapi-checks.yml"), + "version: 1\nchecks:\n core:\n org: parity\n targets: [squads/dental]\n suites: [core]\n again:\n org: parity\n targets: [squads/dental]\n suites: [core]\n", + ); + await withGitHubEnv(stub, join(root, "summary.md"), () => + run(["--all"], root), + ); + const contexts = stub.statuses.map((s) => `${s.context} ${s.state}`); + const aggregate = stub.statuses.at(-1); + assert.deepEqual( + { + perTarget: contexts.filter((c) => c.includes(" / ")).sort(), + aggregate: [aggregate?.context, aggregate?.state], + aggregateLinksWorkflow: + aggregate?.target_url?.includes("/actions/runs/"), + }, + { + perTarget: [ + "Vapi Simulations / again / squads/dental pending", + "Vapi Simulations / again / squads/dental success", + "Vapi Simulations / core / squads/dental pending", + "Vapi Simulations / core / squads/dental success", + ], + aggregate: ["Vapi Simulations", "success"], + aggregateLinksWorkflow: true, + }, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); + test("a failing live run exits 1 and turns the aggregate red", async () => { await withLiveStub(async (stub) => { stub.mode.failFirstItem = true; @@ -341,7 +369,7 @@ test("a failing live run exits 1 and turns the aggregate red", async () => { [ 1, { - context: "Vapi Evals", + context: "Vapi Simulations", state: "failure", description: "1 failed", target_url: "https://dashboard.vapi.ai/run-1", @@ -364,8 +392,8 @@ test("a single named check never posts the aggregate", async () => { assert.deepEqual( stub.statuses.map((s) => s.context), [ - "Vapi Evals / core / squads/dental", - "Vapi Evals / core / squads/dental", + "Vapi Simulations / core / squads/dental", + "Vapi Simulations / core / squads/dental", ], ); } finally { @@ -404,7 +432,13 @@ test("an unaffected change runs nothing and posts a green aggregate", async () = [ 0, 0, - [["Vapi Evals", "success", "No checks affected by this change"]], + [ + [ + "Vapi Simulations", + "success", + "No checks affected by this change", + ], + ], ], ); } finally { @@ -426,7 +460,7 @@ test("a dry run of an affected PR (fork or Dependabot) posts an error aggregate stub.vapiPosts, stub.statuses.map((s) => [s.context, s.state]), ], - [0, 0, [["Vapi Evals", "error"]]], + [0, 0, [["Vapi Simulations", "error"]]], ); } finally { rmSync(root, { recursive: true, force: true }); @@ -447,7 +481,7 @@ test("an invalid vapi-checks.yml still posts an error aggregate", async () => { result.code, stub.statuses.map((s) => [s.context, s.state, s.description]), ], - [2, [["Vapi Evals", "error", "vapi-checks.yml is invalid"]]], + [2, [["Vapi Simulations", "error", "vapi-checks.yml is invalid"]]], ); } finally { rmSync(root, { recursive: true, force: true }); diff --git a/tests/check-report.test.ts b/tests/check-report.test.ts index 9be4a71..d0363a8 100644 --- a/tests/check-report.test.ts +++ b/tests/check-report.test.ts @@ -69,7 +69,7 @@ test("the markdown report has a summary row per target and detail for anything n dryRun: false, }), [ - "## Vapi Evals", + "## Vapi Simulations", "", "| Check | Target | Result | Simulations | Run |", "|---|---|---|---|---|", diff --git a/tests/check-status.test.ts b/tests/check-status.test.ts index d4bbfda..c19643f 100644 --- a/tests/check-status.test.ts +++ b/tests/check-status.test.ts @@ -95,7 +95,7 @@ test("statuses post to the head SHA with the run link, and long descriptions are "POST /repos/acme/gitops/statuses/abc", "Bearer t", { - context: "Vapi Evals / core / squads/main", + context: "Vapi Simulations / core / squads/main", state: "failure", description: 140, target_url: "https://run", @@ -110,7 +110,7 @@ test("a rejected status (read-only token) only warns", async () => { assert.equal( await commitStatusPost( { token: "t", repo: "acme/gitops", sha: "abc", apiUrl }, - { context: "Vapi Evals", state: "success", description: "ok" }, + { context: "Vapi Simulations", state: "success", description: "ok" }, ), false, );