Merge develop into master for 2.0.0

.github/actions/README.md

@@ -0,0 +1,151 @@
+# pluginval GitHub Actions
+
+Two composite actions for using pluginval from another repository's workflows.
+Their `action.yml` files are in `validate/` and `acceptance/` at the repository
+root, which keeps the references short; the scripts they run are in this directory.
+Both run on Linux, macOS and Windows runners, download the matching pluginval
+release (or use a binary you give them), run everything they are given and fail
+the step if anything failed.
+
+Results are shown in two places: the job summary gets a table with a row per
+plugin / config and its test count, followed by the name and message of every
+failed test; and each failed plugin / config gets an error annotation naming its
+failed tests, which also appears in the pull request's checks.
+
+On Linux the plugin is run under `xvfb-run` when there is no `DISPLAY` (install
+`xvfb` on runners that don't have it).
+
+## validate
+
+Validates one or more plugins.
+
+```yaml
+- uses: Tracktion/pluginval/validate@v2
+  with:
+    plugins: |
+      build/MyPlugin_artefacts/Release/VST3/MyPlugin.vst3
+      build/MyPlugin_artefacts/Release/AU/MyPlugin.component
+    strictness-level: 10
+    args: --skip-gui-tests
+```
+
+| Input | Default | |
+| --- | --- | --- |
+| `plugins` | (required) | The plugins to validate, one per line. Each is a path, a glob or an AU id. A glob that matches nothing fails. |
+| `strictness-level` | `5` | The strictness level (1-10). |
+| `args` | | Extra options for `pluginval validate`, split on whitespace. |
+| `report-dir` | | Directory to write the JUnit XML reports to (a temporary directory if empty). |
+| `pluginval-version` | `v2.0.0-beta3` | The pluginval release to download: a tag, or `latest`. |
+| `pluginval-path` | | An existing pluginval binary to use instead of downloading one. |
+
+Every `pluginval validate` option can also be set as an environment variable on
+the step (e.g. `env: { TIMEOUT_MS: 60000 }`), which is the way to pass values
+containing spaces.
+
+To skip tests, set `DISABLED_TESTS` to their names, comma-separated (or to an
+absolute path to a file listing one name per line). Test names contain spaces,
+so they can't be given as `--disabled-tests` in `args`. For example, at strictness
+8 and above "Discrete parameter state restoration" fails for plugins using
+`juce::AudioParameterBool`, until [JUCE snaps its value to 0 or 1](https://forum.juce.com/t/pluginval-can-set-an-audioparameterbools-to-odd-values/50971):
+
+```yaml
+- uses: Tracktion/pluginval/validate@v2
+  env:
+    DISABLED_TESTS: Discrete parameter state restoration
+  with:
+    plugins: build/MyPlugin_artefacts/Release/VST3/MyPlugin.vst3
+    strictness-level: 10
+```
+
+Run `pluginval strictness-help <level>` to list the test names.
+
+## acceptance
+
+Runs the [acceptance tests](../../docs/Acceptance%20testing.md) for every config
+(`*.json`) found recursively under a directory, optionally against one built
+plugin.
+
+```yaml
+- uses: Tracktion/pluginval/acceptance@v2
+  with:
+    config-dir: tests/acceptance
+    plugin: build/MyPlugin_artefacts/Release/VST3/MyPlugin.vst3
+```
+
+| Input | Default | |
+| --- | --- | --- |
+| `config-dir` | (required) | Directory searched recursively for configs. |
+| `plugin` | | Plugin path or AU id to test every config against, instead of the `plugin` in each config. |
+| `record-missing` | `false` | Record a missing reference and pass, instead of failing. |
+| `timeout-ms` | | Fail a test if it takes longer than this (pluginval's default if empty). |
+| `report-dir` | | Directory to write the JUnit XML reports to (a temporary directory if empty). |
+| `pluginval-version` | `v2.0.0-beta3` | The pluginval release to download: a tag, or `latest`. |
+| `pluginval-path` | | An existing pluginval binary to use instead of downloading one. |
+
+Notes:
+
+- Every `*.json` under `config-dir` is treated as a config, apart from the
+  `<reference>.wav.json` manifests. Keep other JSON files (and plugin bundles,
+  which contain their own) out of it.
+- A config holding an array of tests is reported as a failure: use one config
+  per file.
+- Relative `plugin` paths resolve against the working directory; the input,
+  state and reference paths in a config resolve against the config's directory.
+
+## Outputs
+
+Both actions set:
+
+| Output | |
+| --- | --- |
+| `passed` / `failed` | How many plugins / configs passed and failed. |
+| `report-dir` | A directory holding a JUnit XML report for each one. For `acceptance` it also has a `diffs` directory with the `*-diff.wav` files found under `config-dir`. |
+
+To keep the reports and diffs from a failed run:
+
+```yaml
+- uses: Tracktion/pluginval/acceptance@v2
+  id: acceptance
+  with:
+    config-dir: tests/acceptance
+    plugin: build/MyPlugin_artefacts/Release/VST3/MyPlugin.vst3
+
+- uses: actions/upload-artifact@v4
+  if: always()
+  with:
+    name: pluginval-acceptance-${{ runner.os }}
+    path: ${{ steps.acceptance.outputs.report-dir }}
+```
+
+### Per-test results as a pull request check
+
+The actions report failures through the job summary and annotations, which need
+no extra permissions. If you would rather have every pluginval test listed as its
+own check on the pull request, feed the JUnit reports to a reporter action such
+as [mikepenz/action-junit-report](https://github.com/mikepenz/action-junit-report)
+or [dorny/test-reporter](https://github.com/dorny/test-reporter):
+
+```yaml
+jobs:
+  validate:
+    runs-on: macos-latest
+    permissions:
+      contents: read
+      checks: write
+    steps:
+      # ... build the plugin ...
+
+      - uses: Tracktion/pluginval/validate@v2
+        with:
+          plugins: build/MyPlugin_artefacts/Release/VST3/MyPlugin.vst3
+          report-dir: pluginval-reports
+
+      - uses: mikepenz/action-junit-report@v5
+        if: always()
+        with:
+          report_paths: pluginval-reports/*.xml
+          check_name: pluginval
+```
+
+This needs the `checks: write` permission, which workflows triggered by pull
+requests from forks don't get, so on those the reporter step can't publish.

.github/actions/acceptance/acceptance.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+# Runs "pluginval test" on every config found (recursively) under
+# INPUT_CONFIG_DIR, runs them all, and fails if any failed. See action.yml for
+# the inputs.
+
+set -u
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/../common/common.sh"
+
+resolve_pluginval
+
+config_dir="${INPUT_CONFIG_DIR:-}"
+
+if [ ! -d "$config_dir" ]; then
+    echo "::error::config-dir is not a directory: $config_dir"
+    exit 1
+fi
+
+report_dir="${INPUT_REPORT_DIR:-$RUNNER_TEMP/pluginval-reports/acceptance}"
+mkdir -p "$report_dir"
+
+options=()
+[ -n "${INPUT_PLUGIN:-}" ] && options+=(--plugin "$INPUT_PLUGIN")
+[ "${INPUT_RECORD_MISSING:-false}" = "true" ] && options+=(--record-missing)
+[ -n "${INPUT_TIMEOUT_MS:-}" ] && options+=(--timeout-ms "$INPUT_TIMEOUT_MS")
+
+passed=0
+failed=0
+index=0
+rows=""
+details=""
+
+# Every *.json is a config, except the <reference>.wav.json manifests that
+# pluginval writes next to each reference.
+IFS=$'\n'
+configs=( $(find "$config_dir" -type f -name '*.json' ! -name '*.wav.json' | sort) )
+unset IFS
+
+for config in ${configs[@]+"${configs[@]}"}; do
+    index=$((index + 1))
+    name="${config#"$config_dir"/}"
+    report="$report_dir/$index-$(sanitise "$(basename "$config" .json)").xml"
+    rm -f "$report"   # so an earlier run's report can't be read if this one writes none
+
+    echo "::group::pluginval test: $name"
+
+    # pluginval only runs the first entry of an array config, so reject them
+    # rather than silently skipping tests.
+    if [ "$(tr -d '[:space:]' < "$config" | head -c 1)" = "[" ]; then
+        failure="Array configs aren't supported: use one config per file."
+        echo "$failure"
+        code=1
+    else
+        run_pluginval test ${options[@]+"${options[@]}"} --junit-xml "$report" "$config"
+        code=$?
+        failure="pluginval exited with code $code (see the log)"
+    fi
+
+    echo "::endgroup::"
+
+    # The per-test results come from the JUnit report; pass / fail from the exit code.
+    read_report "$report"
+
+    if [ $code -eq 0 ]; then
+        passed=$((passed + 1))
+        rows="$rows| \`$name\` | :white_check_mark: passed | $REPORT_TESTS |"$'\n'
+    else
+        failed=$((failed + 1))
+        rows="$rows| \`$name\` | :x: failed | $REPORT_TESTS |"$'\n'
+        [ -z "$REPORT_DETAILS" ] && REPORT_DETAILS="  - $failure"$'\n'
+        details="$details- \`$name\`"$'\n'"$REPORT_DETAILS"
+        annotate_error "pluginval acceptance test failed: $name" "${REPORT_NAMES:-$failure}"
+    fi
+done
+
+if [ $index -eq 0 ]; then
+    echo "::error::No configs (*.json) found under $config_dir"
+    exit 1
+fi
+
+# Gather the diff WAVs written next to failing references, so they can be
+# uploaded along with the reports.
+find "$config_dir" -type f -name '*-diff.wav' | while IFS= read -r diff; do
+    mkdir -p "$report_dir/diffs"
+    cp "$diff" "$report_dir/diffs/$(sanitise "${diff#"$config_dir"/}")"
+done
+
+{
+    echo "### pluginval acceptance tests"
+    echo
+    echo "| Config | Result | Tests |"
+    echo "| --- | --- | --- |"
+    printf '%s' "$rows"
+
+    if [ -n "$details" ]; then
+        echo
+        echo "#### Failed tests"
+        echo
+        printf '%s' "$details"
+    fi
+} >> "$GITHUB_STEP_SUMMARY"
+
+{
+    echo "passed=$passed"
+    echo "failed=$failed"
+    echo "report-dir=$report_dir"
+} >> "$GITHUB_OUTPUT"
+
+echo "pluginval acceptance tests: $passed passed, $failed failed"
+[ $failed -eq 0 ]

.github/actions/common/common.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Shared helpers for the pluginval composite actions (sourced, not run).
+# Kept compatible with bash 3.2, which is what macOS runners ship.
+
+COMMON_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Defaults so the scripts can also be run outside of GitHub Actions.
+if [ -z "${RUNNER_OS:-}" ]; then
+    if [ "$(uname -s)" = "Darwin" ]; then RUNNER_OS=macOS
+    elif [ "$(uname -s)" = "Linux" ]; then RUNNER_OS=Linux
+    else RUNNER_OS=Windows
+    fi
+fi
+
+: "${RUNNER_TEMP:=$(mktemp -d)}"
+: "${GITHUB_STEP_SUMMARY:=/dev/null}"
+: "${GITHUB_OUTPUT:=/dev/null}"
+
+# Sets PLUGINVAL_BIN to the pluginval binary: INPUT_PLUGINVAL_PATH if given,
+# otherwise the INPUT_PLUGINVAL_VERSION release ("latest" or a tag like v2.0.0-beta1)
+# downloaded for this runner's OS.
+resolve_pluginval()
+{
+    if [ -n "${INPUT_PLUGINVAL_PATH:-}" ]; then
+        PLUGINVAL_BIN="$INPUT_PLUGINVAL_PATH"
+    else
+        local version="${INPUT_PLUGINVAL_VERSION:-latest}"
+        local dir="$RUNNER_TEMP/pluginval-$version"
+        local url="https://github.com/Tracktion/pluginval/releases/download/$version/pluginval_$RUNNER_OS.zip"
+
+        if [ "$version" = "latest" ]; then
+            url="https://github.com/Tracktion/pluginval/releases/latest/download/pluginval_$RUNNER_OS.zip"
+        fi
+
+        case "$RUNNER_OS" in
+            macOS)   PLUGINVAL_BIN="$dir/pluginval.app/Contents/MacOS/pluginval" ;;
+            Windows) PLUGINVAL_BIN="$dir/pluginval.exe" ;;
+            *)       PLUGINVAL_BIN="$dir/pluginval" ;;
+        esac
+
+        if [ ! -f "$PLUGINVAL_BIN" ]; then
+            echo "Downloading $url"
+            mkdir -p "$dir"
+
+            if ! curl -fsSL --retry 3 "$url" -o "$dir/pluginval.zip"; then
+                echo "::error::Couldn't download pluginval $version for $RUNNER_OS ($url)"
+                exit 1
+            fi
+
+            unzip -q -o "$dir/pluginval.zip" -d "$dir"
+            chmod +x "$PLUGINVAL_BIN"
+        fi
+    fi
+
+    if [ ! -f "$PLUGINVAL_BIN" ]; then
+        echo "::error::pluginval binary not found: $PLUGINVAL_BIN"
+        exit 1
+    fi
+}
+
+# Runs pluginval with the given arguments. On Linux without a display it is
+# wrapped in xvfb-run, as plugins generally need an X server to load.
+run_pluginval()
+{
+    if [ "$RUNNER_OS" = "Linux" ] && [ -z "${DISPLAY:-}" ] && command -v xvfb-run > /dev/null; then
+        xvfb-run --auto-servernum "$PLUGINVAL_BIN" "$@"
+    else
+        "$PLUGINVAL_BIN" "$@"
+    fi
+}
+
+# Prints a file-name-safe version of the argument.
+sanitise()
+{
+    printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '_'
+}
+
+# Reads a JUnit XML report written by pluginval and sets:
+#   REPORT_TESTS    a summary-table cell, e.g. "17" or "2 of 17 failed"
+#   REPORT_NAMES    "name: message" per failed test, "; " separated (for the annotation)
+#   REPORT_DETAILS  a markdown list item per failed test (for the job summary)
+# All three are empty if there is no report.
+read_report()
+{
+    REPORT_TESTS=""
+    REPORT_NAMES=""
+    REPORT_DETAILS=""
+
+    [ -f "$1" ] || return 0
+
+    local kind first second
+    while IFS=$'\t' read -r kind first second; do
+        if [ "$kind" = "T" ]; then
+            if [ "$second" -gt 0 ]; then REPORT_TESTS="$second of $first failed"; else REPORT_TESTS="$first"; fi
+        elif [ "$kind" = "F" ]; then
+            REPORT_NAMES="${REPORT_NAMES:+$REPORT_NAMES; }$first: $second"
+            REPORT_DETAILS="$REPORT_DETAILS  - **${first//|/\\|}**: ${second//|/\\|}"$'\n'
+        fi
+    done < <(awk -f "$COMMON_DIR/junit.awk" "$1")
+}
+
+# Prints an ::error annotation, escaping the message as workflow commands need.
+annotate_error()
+{
+    local message="${2//%/%25}"
+    echo "::error title=$1::$message"
+}