diff --git a/.github/BRANCH_PROTECTION.md b/.github/BRANCH_PROTECTION.md index 705c011..6e5ca5d 100644 --- a/.github/BRANCH_PROTECTION.md +++ b/.github/BRANCH_PROTECTION.md @@ -14,11 +14,12 @@ policy so it can be re-applied or audited. |---|---| | `clang-format` | now | | `testserver` | now | - | `bootstrap-script` | now | - | `build (gcc)` / `build (clang)` | when the first C++ lane merges | - | `sanitizers (dev)` / `sanitizers (tsan)` | when the first C++ lane merges | + | `bootstrap-script` | now (validates package names against the 24.04 runner archive) | + | `bootstrap-script-2604` | now — real `--with-clang` install in a 26.04 container; the release the project ships on | + | `build (gcc)` / `build (clang)` | now — core, daemon and gui have merged | + | `sanitizers (dev)` / `sanitizers (tsan)` | now — core, daemon and gui have merged | | `conformance` | **now — `tests/conformance/` has landed; this is the M0 exit gate** | - | `extension-lint` | when `extension/` lands | + | `extension-lint` | now — `extension/` has merged (MV3 manifest + esbuild build) | `clang-tidy` is intentionally **not** required through M1 (`continue-on-error: true`, `.clang-tidy` has `WarningsAsErrors: ''`). Make it required at M2. @@ -39,5 +40,7 @@ Their guards fail **loudly** (non-zero) once the lane is half-present — e.g. a filename is how a required check ends up green over nothing; the skip branch is only for a lane that is genuinely absent. -`conformance` no longer has a skip branch. The suite (`tests/conformance/run.sh`) has -landed, so the job runs it unconditionally and fails if the entrypoint is missing. +`conformance` has no skip branch. It runs `ctest -L conformance` (see +`docs/adr/0014-conformance-runs-through-ctest.md`); the `dev` test preset's +`noTestsAction: error` fails the job if that label ever matches nothing, so a deleted or +renamed registration goes red instead of passing vacuously. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8479275..04446fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,12 +41,38 @@ jobs: run: python3 tools/testserver/selftest.py bootstrap-script: - # Keeps tools/bootstrap.sh honest: it must run clean and its --check must pass. + # Keeps tools/bootstrap.sh honest on the runner image: it must run clean and its + # --check must pass. ubuntu-latest is 24.04; the project ships on 26.04, so this + # exercises the 24.04 archive only. bootstrap-script-2604 below is what validates + # the package names against the release the project actually targets. runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - run: sudo ./tools/bootstrap.sh --with-clang - - run: ./tools/bootstrap.sh --check + - run: ./tools/bootstrap.sh --check --with-clang + # Cheap: apt-cache only. Validates the M6 packaging names now so they can't rot + # unnoticed until M6. + - run: ./tools/bootstrap.sh --check --with-clang --packaging + + bootstrap-script-2604: + # The project targets 26.04 and GitHub has no 26.04 runner image yet, so the one + # automated place bootstrap.sh runs is on the wrong release to catch a name that is + # valid on 24.04 and gone on 26.04 — which is exactly how libqt6svg6-dev reached a + # contributor's VM (gui/docs/pkg-qa-requests-m1.md R1/R2). Run the real install in a + # 26.04 container, with --with-clang: the fuzz toolchain had never been exercised + # anywhere (CORE ran clang++-21 directly because it can't sudo). + runs-on: ubuntu-latest + container: ubuntu:26.04 + steps: + - name: Base tools for checkout + run: | + apt-get update -qq + apt-get install -y --no-install-recommends ca-certificates git sudo + - uses: actions/checkout@v4 + - name: Full bootstrap on 26.04 (--with-clang) + run: ./tools/bootstrap.sh --with-clang + - name: Re-verify + run: ./tools/bootstrap.sh --check --with-clang extension-lint: runs-on: ubuntu-latest @@ -105,7 +131,10 @@ jobs: - name: Build run: cmake --build --preset ci - name: Test - run: ctest --preset ci --output-on-failure + # -E '^conformance$' drops the end-to-end run.sh test (npm installs, its own + # mockd, ~24 s); the dedicated `conformance` job owns that one run. The native + # `conformance_cpp` test is not excluded and still runs on every matrix leg. + run: ctest --preset ci --output-on-failure -E '^conformance$' sanitizers: runs-on: ubuntu-latest @@ -122,7 +151,10 @@ jobs: - name: Build run: cmake --build --preset ${{ matrix.preset }} - name: Test - run: ctest --preset ${{ matrix.preset }} --output-on-failure + # See the build job: the end-to-end run.sh test is the dedicated `conformance` + # job's; sanitizing a suite that shells out to its own unsanitized g++ build and + # a node process buys nothing. `conformance_cpp` still runs here under the sanitizer. + run: ctest --preset ${{ matrix.preset }} --output-on-failure -E '^conformance$' env: ASAN_OPTIONS: detect_leaks=1:halt_on_error=1 UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 @@ -164,24 +196,27 @@ jobs: # against the other. Required on every PR — branch protection is a repo setting, # recorded in .github/BRANCH_PROTECTION.md. # - # The suite ships as tests/conformance/run.sh (its own mockd, its own C++ build). - # There is no CMakeLists or package.json to guard on, and no ctest label to select; - # the earlier guard for those was always false, so this job passed running nothing. + # Canonical entry point is `ctest -L conformance`. tests/conformance/CMakeLists.txt + # (owned by PROTO) registers two tests under that label: `conformance`, which shells + # out to run.sh end to end, and `conformance_cpp`, the finer-grained native runner. + # CI drives it exactly as a developer does — one definition of "the suite passed", + # and PROTO's registration is on the exercised path so it cannot rot. See + # docs/adr/0014-conformance-runs-through-ctest.md. + # + # `noTestsAction: error` in the dev test preset is the rot guard: if the label ever + # matches nothing (registration deleted, typo), ctest exits non-zero instead of + # passing vacuously. runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Guard — the suite must be present - run: | - if [ ! -x tests/conformance/run.sh ]; then - echo "::error::tests/conformance/run.sh is missing or not executable. The" \ - "conformance suite is the M0 exit gate and this check must not pass" \ - "without running it. If PROTO moved the entrypoint, update this job." - exit 1 - fi - name: Bootstrap toolchain run: sudo ./tools/bootstrap.sh - uses: actions/setup-node@v4 with: - node-version: '22' # apt ships < 20; run.sh's TS runner needs >= 20 - - name: Run conformance suite - run: ./tests/conformance/run.sh + node-version: '22' # apt ships < 20; run.sh's TS replay runner needs >= 20 + - name: Configure + run: cmake --preset dev + - name: Build the native conformance runner + run: cmake --build --preset dev --target velox_conformance_cpp + - name: Run conformance (ctest -L conformance) + run: ctest --preset dev -L conformance --output-on-failure diff --git a/.gitignore b/.gitignore index 1ba717e..9701178 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,11 @@ massif.out.* *.log *.veloxpart *.veloxpart.meta + +# Fuzzing crash artifacts — libFuzzer writes these to CWD on a find and each holds the +# crashing input verbatim. Ignore so they are never committed by accident (CORE caught +# two by hand before this rule existed). +crash-* +oom-* +leak-* +timeout-* diff --git a/README.md b/README.md index 3471ebd..c458bae 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ Surveyed on this machine 2026-09-09 — **most of it is already installed**: ```bash sudo apt update && sudo apt install -y \ - libqt6svg6-dev \ # GUI: SVG icon rendering + qt6-svg-dev \ # GUI: SVG icon rendering libsecret-1-dev \ # DAEMON: Secret Service for site logins nodejs npm \ # EXT + PROTO: extension build, mockd, conformance runner clang # optional: libFuzzer targets in M7 diff --git a/docs/adr/0014-conformance-runs-through-ctest.md b/docs/adr/0014-conformance-runs-through-ctest.md new file mode 100644 index 0000000..2106c5c --- /dev/null +++ b/docs/adr/0014-conformance-runs-through-ctest.md @@ -0,0 +1,50 @@ +# ADR 0014 — The conformance suite has one entry point: `ctest -L conformance` + +**Status:** accepted · **Date:** 2026-09-10 · **Lane:** PKG/QA + +## Context + +The conformance suite could be started two ways, and both were live: + +* `tests/conformance/run.sh` — the script a developer runs by hand; the CI `conformance` + job called it directly. +* `ctest -L conformance` — PROTO wired `tests/conformance/CMakeLists.txt` to register + `conformance` (which shells out to `run.sh`) and `conformance_cpp` (the finer-grained + native runner) under that label. + +Nothing was wrong with either, but the CI `conformance` job — the one named as the M0 +exit gate in `.github/BRANCH_PROTECTION.md` — exercised only the script. The ctest +registration was reached only incidentally, by the unfiltered `ctest` run in the `build` +and `sanitizers` jobs. A change that broke the registration (a deleted `add_test`, a +renamed label, a `WORKING_DIRECTORY` regression) would not fail the check that is supposed +to be about conformance; it would fail a build job, or nothing. + +Two entry points also means two answers to "did the suite pass", free to drift. + +## Decision + +`ctest -L conformance` is the canonical entry point. `run.sh` remains the thing it runs — +`tests/conformance/CMakeLists.txt` registers it verbatim — but CI reaches it only through +ctest, the same way a developer does. + +* The `conformance` job runs `cmake --preset dev`, builds just the `velox_conformance_cpp` + target, then `ctest --preset dev -L conformance --output-on-failure`. This puts PROTO's + registration on the exercised path. +* The rot guard is `noTestsAction: error` in the `dev` test preset: if the label ever + matches zero tests, ctest exits non-zero instead of the old silent "0 tests, exit 0". +* The `build` and `sanitizers` jobs exclude the end-to-end test with `-E '^conformance$'`. + It installs two npm trees and runs a node process against a self-compiled, unsanitized + `g++` binary — running that once, in the dedicated job, is enough; sanitizing it buys + nothing. `conformance_cpp` is *not* excluded and still runs on every matrix leg, which + is where ASan/UBSan/TSan coverage of the generated wire code is worth having. + +## Consequences + +* One definition of "the suite passed". +* `run.sh` still works standalone and unchanged for local use and for + `run.sh --uds … --ws-port …` against a real daemon; that path is out of scope for this + ADR, which is only about what CI treats as authoritative. +* If PROTO ever wants the two labelled tests separated (e.g. a `conformance-e2e` label so + jobs can select them independently without a name regex), that is a `tests/conformance/` + change on their side; the `-E '^conformance$'` here is the PKG/QA-side stopgap until + then. diff --git a/docs/agents/AGENT-PKG-QA.md b/docs/agents/AGENT-PKG-QA.md index 9f5f531..c9fda26 100644 --- a/docs/agents/AGENT-PKG-QA.md +++ b/docs/agents/AGENT-PKG-QA.md @@ -14,7 +14,7 @@ Do not write feature code in any lane's directory. 1. **Toolchain script** — `tools/bootstrap.sh` installing the `apt` list in the README, verified on a clean 26.04 VM. The dev machine already has git 2.53, CMake 4.2.3, g++ 15.2, ninja, Qt 6 dev, libcurl, SQLite, nlohmann-json and ffmpeg; only - `libqt6svg6-dev`, `libsecret-1-dev`, `nodejs`/`npm` and (optionally) `clang` are + `qt6-svg-dev`, `libsecret-1-dev`, `nodejs`/`npm` and (optionally) `clang` are missing. The script must still install the full list for clean machines. ⚠ CMake 4.x rejects `cmake_minimum_required` below 3.5 — check every dependency. 2. **CMake** — top-level `CMakeLists.txt` + `CMakePresets.json` with presets: diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 0000000..f177ce8 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,95 @@ +# tests/integration — cross-lane runtime gates + +Owned by PKG/QA. Real binaries against `tools/mockd` / `tools/testserver`, headless. +Unit tests live in each lane; this tree is for behaviour that only shows up when the +pieces run together (throughput, memory over time, reconnect). + +## GUI M1 definition-of-done gates (R3) + +`gui/docs/pkg-qa-requests-m1.md` R3: three GUI DoD items are not unit tests and have +nowhere to run. GUI owns the harness; PKG/QA owns the CI job. This is the wiring contract +so the two halves meet without another round trip. + +### What PKG/QA needs from GUI + +A driver invoked as `gui/tests/dod/run.sh [--json ]` (exact path TBD by GUI), +headless-capable (Xvfb or offscreen `QT_QPA_PLATFORM`), with: + +| `` | Pass / fail condition | Budget | +|---|---|---| +| `scroll-60fps` | `mockd --tasks 10000`, scripted fling scroll; **fail** if p99 frame > 16.6 ms | per-PR | +| `rss-flat` | `mockd --tasks 10000` + progress events, 10 min; **fail** if RSS growth > a fixed slack (GUI picks the number, states it) | nightly | +| `unhappy-path` | `mockd --slow` / `--flaky ` / `--drop-connection `; **fail** on crash, on watchdog-detected hang, or if connection state never returns to `Connected` | per-PR | + +Contract: + +* exit `0` pass, non-zero fail; a hang is the harness's own watchdog to catch and turn + into a non-zero exit, not something CI should have to `timeout(1)` around. +* `--json` writes one machine-readable result file (measured p99, RSS series, recovery + time) so the job can upload it as an artifact and a regression is a diff, not a re-run. +* no network, no writes outside a tempdir, no leaked child processes on failure. + +### CI job — pre-drafted, add once the harness path is fixed + +```yaml + gui-dod: + # Per-PR GUI gates. The 10-minute rss-flat gate is in gui-dod-nightly, not here. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Bootstrap toolchain + run: sudo ./tools/bootstrap.sh + - uses: actions/setup-node@v4 + with: + node-version: '22' # tools/mockd + - name: Configure + build + run: | + cmake --preset dev + cmake --build --preset dev --target velox-gui + - name: Install mockd + run: cd tools/mockd && npm ci + - name: Xvfb + gates + run: | + sudo apt-get install -y --no-install-recommends xvfb + xvfb-run -a gui/tests/dod/run.sh scroll-60fps --json scroll.json # TODO(GUI): path + xvfb-run -a gui/tests/dod/run.sh unhappy-path --json unhappy.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: gui-dod-${{ github.run_id }} + path: "*.json" + + gui-dod-nightly: + if: github.event_name == 'schedule' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Bootstrap toolchain + run: sudo ./tools/bootstrap.sh + - uses: actions/setup-node@v4 + with: + node-version: '22' + - name: Configure + build + run: | + cmake --preset dev + cmake --build --preset dev --target velox-gui + - run: cd tools/mockd && npm ci + - name: RSS soak (10 min) + run: | + sudo apt-get install -y --no-install-recommends xvfb + xvfb-run -a gui/tests/dod/run.sh rss-flat --json rss.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: gui-dod-rss-${{ github.run_id }} + path: rss.json +``` + +`gui-dod-nightly` needs a `schedule:` trigger added to the top of `ci.yml` (there is none +today), or its own workflow file. Decide that when the harness lands. + +### Status + +Blocked on GUI's harness. Not urgent (GUI M1 DoD, not M0). When GUI files the follow-up +with the real `run.sh` path and the `rss-flat` slack number, PKG/QA drops the `TODO(GUI)` +markers, adds the `schedule:` trigger, and marks `gui-dod` required. diff --git a/tools/bootstrap.sh b/tools/bootstrap.sh index d032623..49fc17d 100755 --- a/tools/bootstrap.sh +++ b/tools/bootstrap.sh @@ -84,7 +84,9 @@ APT_GUI=( qt6-base-dev qt6-tools-dev qt6-tools-dev-tools # lrelease/lupdate for i18n - libqt6svg6-dev # GUI: SVG icon rendering + qt6-svg-dev # GUI: SVG icon rendering. Was libqt6svg6-dev, which has no + # candidate on 26.04; qt6-svg-dev is the package that + # ships the headers + Qt6SvgConfig.cmake find_package needs. ) APT_LINT=( @@ -144,6 +146,36 @@ need_cmd() { fi } +# Every apt name this script would install must resolve to an installable candidate on +# THIS release. The checks below verify outcomes — binaries on $PATH, pkg-config modules — +# on a box that already installed everything; none of them look at the APT_* names, which +# is the one part of the script that breaks on a clean machine of the wrong release +# (libqt6svg6-dev did exactly this on 26.04). See gui/docs/pkg-qa-requests-m1.md R2. +# `apt-cache policy` needs no root and no network; it reads the lists apt already has. +log "validating apt package names (${#PKGS[@]})" +missing_pkgs=() +for pkg in "${PKGS[@]}"; do + # `apt-cache policy` exits 0 whether or not the name resolves; a name with no + # installable version prints "Candidate: (none)". Grep the line, don't trust $?. + cand="$(apt-cache policy "$pkg" 2>/dev/null | sed -n 's/^ Candidate: //p')" + if [[ -z "$cand" || "$cand" == "(none)" ]]; then + missing_pkgs+=("$pkg") + fi +done +if (( ${#missing_pkgs[@]} == 0 )); then + printf ' \033[32m✓\033[0m %-16s all %d resolve on %s\n' \ + "apt names" "${#PKGS[@]}" "${VERSION_ID:-this release}" +elif (( ${#missing_pkgs[@]} == ${#PKGS[@]} )); then + warn "no apt candidate for ANY of the ${#PKGS[@]} packages — the apt lists are almost" + warn "certainly stale, not the package set. Run 'apt-get update' and re-run --check." +else + for pkg in "${missing_pkgs[@]}"; do + printf ' \033[31m✗\033[0m %-16s no installable candidate on %s\n' \ + "$pkg" "${VERSION_ID:-this release}" + done + fail=1 +fi + need_cmd git git --version need_cmd cmake cmake --version need_cmd ninja ninja --version