merge: lane/pkg-qa
This commit is contained in:
@@ -14,11 +14,12 @@ policy so it can be re-applied or audited.
|
||||
|---|---|
|
||||
| `clang-format` | now |
|
||||
| `testserver` | now |
|
||||
| `bootstrap-script` | now |
|
||||
| `build (gcc)` / `build (clang)` | when the first C++ lane merges |
|
||||
| `sanitizers (dev)` / `sanitizers (tsan)` | when the first C++ lane merges |
|
||||
| `bootstrap-script` | now (validates package names against the 24.04 runner archive) |
|
||||
| `bootstrap-script-2604` | now — real `--with-clang` install in a 26.04 container; the release the project ships on |
|
||||
| `build (gcc)` / `build (clang)` | now — core, daemon and gui have merged |
|
||||
| `sanitizers (dev)` / `sanitizers (tsan)` | now — core, daemon and gui have merged |
|
||||
| `conformance` | **now — `tests/conformance/` has landed; this is the M0 exit gate** |
|
||||
| `extension-lint` | when `extension/` lands |
|
||||
| `extension-lint` | now — `extension/` has merged (MV3 manifest + esbuild build) |
|
||||
|
||||
`clang-tidy` is intentionally **not** required through M1 (`continue-on-error: true`,
|
||||
`.clang-tidy` has `WarningsAsErrors: ''`). Make it required at M2.
|
||||
@@ -39,5 +40,7 @@ Their guards fail **loudly** (non-zero) once the lane is half-present — e.g. a
|
||||
filename is how a required check ends up green over nothing; the skip branch is only for
|
||||
a lane that is genuinely absent.
|
||||
|
||||
`conformance` no longer has a skip branch. The suite (`tests/conformance/run.sh`) has
|
||||
landed, so the job runs it unconditionally and fails if the entrypoint is missing.
|
||||
`conformance` has no skip branch. It runs `ctest -L conformance` (see
|
||||
`docs/adr/0014-conformance-runs-through-ctest.md`); the `dev` test preset's
|
||||
`noTestsAction: error` fails the job if that label ever matches nothing, so a deleted or
|
||||
renamed registration goes red instead of passing vacuously.
|
||||
|
||||
+53
-18
@@ -41,12 +41,38 @@ jobs:
|
||||
run: python3 tools/testserver/selftest.py
|
||||
|
||||
bootstrap-script:
|
||||
# Keeps tools/bootstrap.sh honest: it must run clean and its --check must pass.
|
||||
# Keeps tools/bootstrap.sh honest on the runner image: it must run clean and its
|
||||
# --check must pass. ubuntu-latest is 24.04; the project ships on 26.04, so this
|
||||
# exercises the 24.04 archive only. bootstrap-script-2604 below is what validates
|
||||
# the package names against the release the project actually targets.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: sudo ./tools/bootstrap.sh --with-clang
|
||||
- run: ./tools/bootstrap.sh --check
|
||||
- run: ./tools/bootstrap.sh --check --with-clang
|
||||
# Cheap: apt-cache only. Validates the M6 packaging names now so they can't rot
|
||||
# unnoticed until M6.
|
||||
- run: ./tools/bootstrap.sh --check --with-clang --packaging
|
||||
|
||||
bootstrap-script-2604:
|
||||
# The project targets 26.04 and GitHub has no 26.04 runner image yet, so the one
|
||||
# automated place bootstrap.sh runs is on the wrong release to catch a name that is
|
||||
# valid on 24.04 and gone on 26.04 — which is exactly how libqt6svg6-dev reached a
|
||||
# contributor's VM (gui/docs/pkg-qa-requests-m1.md R1/R2). Run the real install in a
|
||||
# 26.04 container, with --with-clang: the fuzz toolchain had never been exercised
|
||||
# anywhere (CORE ran clang++-21 directly because it can't sudo).
|
||||
runs-on: ubuntu-latest
|
||||
container: ubuntu:26.04
|
||||
steps:
|
||||
- name: Base tools for checkout
|
||||
run: |
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends ca-certificates git sudo
|
||||
- uses: actions/checkout@v4
|
||||
- name: Full bootstrap on 26.04 (--with-clang)
|
||||
run: ./tools/bootstrap.sh --with-clang
|
||||
- name: Re-verify
|
||||
run: ./tools/bootstrap.sh --check --with-clang
|
||||
|
||||
extension-lint:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -105,7 +131,10 @@ jobs:
|
||||
- name: Build
|
||||
run: cmake --build --preset ci
|
||||
- name: Test
|
||||
run: ctest --preset ci --output-on-failure
|
||||
# -E '^conformance$' drops the end-to-end run.sh test (npm installs, its own
|
||||
# mockd, ~24 s); the dedicated `conformance` job owns that one run. The native
|
||||
# `conformance_cpp` test is not excluded and still runs on every matrix leg.
|
||||
run: ctest --preset ci --output-on-failure -E '^conformance$'
|
||||
|
||||
sanitizers:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -122,7 +151,10 @@ jobs:
|
||||
- name: Build
|
||||
run: cmake --build --preset ${{ matrix.preset }}
|
||||
- name: Test
|
||||
run: ctest --preset ${{ matrix.preset }} --output-on-failure
|
||||
# See the build job: the end-to-end run.sh test is the dedicated `conformance`
|
||||
# job's; sanitizing a suite that shells out to its own unsanitized g++ build and
|
||||
# a node process buys nothing. `conformance_cpp` still runs here under the sanitizer.
|
||||
run: ctest --preset ${{ matrix.preset }} --output-on-failure -E '^conformance$'
|
||||
env:
|
||||
ASAN_OPTIONS: detect_leaks=1:halt_on_error=1
|
||||
UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
|
||||
@@ -164,24 +196,27 @@ jobs:
|
||||
# against the other. Required on every PR — branch protection is a repo setting,
|
||||
# recorded in .github/BRANCH_PROTECTION.md.
|
||||
#
|
||||
# The suite ships as tests/conformance/run.sh (its own mockd, its own C++ build).
|
||||
# There is no CMakeLists or package.json to guard on, and no ctest label to select;
|
||||
# the earlier guard for those was always false, so this job passed running nothing.
|
||||
# Canonical entry point is `ctest -L conformance`. tests/conformance/CMakeLists.txt
|
||||
# (owned by PROTO) registers two tests under that label: `conformance`, which shells
|
||||
# out to run.sh end to end, and `conformance_cpp`, the finer-grained native runner.
|
||||
# CI drives it exactly as a developer does — one definition of "the suite passed",
|
||||
# and PROTO's registration is on the exercised path so it cannot rot. See
|
||||
# docs/adr/0014-conformance-runs-through-ctest.md.
|
||||
#
|
||||
# `noTestsAction: error` in the dev test preset is the rot guard: if the label ever
|
||||
# matches nothing (registration deleted, typo), ctest exits non-zero instead of
|
||||
# passing vacuously.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Guard — the suite must be present
|
||||
run: |
|
||||
if [ ! -x tests/conformance/run.sh ]; then
|
||||
echo "::error::tests/conformance/run.sh is missing or not executable. The" \
|
||||
"conformance suite is the M0 exit gate and this check must not pass" \
|
||||
"without running it. If PROTO moved the entrypoint, update this job."
|
||||
exit 1
|
||||
fi
|
||||
- name: Bootstrap toolchain
|
||||
run: sudo ./tools/bootstrap.sh
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22' # apt ships < 20; run.sh's TS runner needs >= 20
|
||||
- name: Run conformance suite
|
||||
run: ./tests/conformance/run.sh
|
||||
node-version: '22' # apt ships < 20; run.sh's TS replay runner needs >= 20
|
||||
- name: Configure
|
||||
run: cmake --preset dev
|
||||
- name: Build the native conformance runner
|
||||
run: cmake --build --preset dev --target velox_conformance_cpp
|
||||
- name: Run conformance (ctest -L conformance)
|
||||
run: ctest --preset dev -L conformance --output-on-failure
|
||||
|
||||
@@ -37,3 +37,11 @@ massif.out.*
|
||||
*.log
|
||||
*.veloxpart
|
||||
*.veloxpart.meta
|
||||
|
||||
# Fuzzing crash artifacts — libFuzzer writes these to CWD on a find and each holds the
|
||||
# crashing input verbatim. Ignore so they are never committed by accident (CORE caught
|
||||
# two by hand before this rule existed).
|
||||
crash-*
|
||||
oom-*
|
||||
leak-*
|
||||
timeout-*
|
||||
|
||||
@@ -103,7 +103,7 @@ Surveyed on this machine 2026-09-09 — **most of it is already installed**:
|
||||
|
||||
```bash
|
||||
sudo apt update && sudo apt install -y \
|
||||
libqt6svg6-dev \ # GUI: SVG icon rendering
|
||||
qt6-svg-dev \ # GUI: SVG icon rendering
|
||||
libsecret-1-dev \ # DAEMON: Secret Service for site logins
|
||||
nodejs npm \ # EXT + PROTO: extension build, mockd, conformance runner
|
||||
clang # optional: libFuzzer targets in M7
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# ADR 0014 — The conformance suite has one entry point: `ctest -L conformance`
|
||||
|
||||
**Status:** accepted · **Date:** 2026-09-10 · **Lane:** PKG/QA
|
||||
|
||||
## Context
|
||||
|
||||
The conformance suite could be started two ways, and both were live:
|
||||
|
||||
* `tests/conformance/run.sh` — the script a developer runs by hand; the CI `conformance`
|
||||
job called it directly.
|
||||
* `ctest -L conformance` — PROTO wired `tests/conformance/CMakeLists.txt` to register
|
||||
`conformance` (which shells out to `run.sh`) and `conformance_cpp` (the finer-grained
|
||||
native runner) under that label.
|
||||
|
||||
Nothing was wrong with either, but the CI `conformance` job — the one named as the M0
|
||||
exit gate in `.github/BRANCH_PROTECTION.md` — exercised only the script. The ctest
|
||||
registration was reached only incidentally, by the unfiltered `ctest` run in the `build`
|
||||
and `sanitizers` jobs. A change that broke the registration (a deleted `add_test`, a
|
||||
renamed label, a `WORKING_DIRECTORY` regression) would not fail the check that is supposed
|
||||
to be about conformance; it would fail a build job, or nothing.
|
||||
|
||||
Two entry points also means two answers to "did the suite pass", free to drift.
|
||||
|
||||
## Decision
|
||||
|
||||
`ctest -L conformance` is the canonical entry point. `run.sh` remains the thing it runs —
|
||||
`tests/conformance/CMakeLists.txt` registers it verbatim — but CI reaches it only through
|
||||
ctest, the same way a developer does.
|
||||
|
||||
* The `conformance` job runs `cmake --preset dev`, builds just the `velox_conformance_cpp`
|
||||
target, then `ctest --preset dev -L conformance --output-on-failure`. This puts PROTO's
|
||||
registration on the exercised path.
|
||||
* The rot guard is `noTestsAction: error` in the `dev` test preset: if the label ever
|
||||
matches zero tests, ctest exits non-zero instead of the old silent "0 tests, exit 0".
|
||||
* The `build` and `sanitizers` jobs exclude the end-to-end test with `-E '^conformance$'`.
|
||||
It installs two npm trees and runs a node process against a self-compiled, unsanitized
|
||||
`g++` binary — running that once, in the dedicated job, is enough; sanitizing it buys
|
||||
nothing. `conformance_cpp` is *not* excluded and still runs on every matrix leg, which
|
||||
is where ASan/UBSan/TSan coverage of the generated wire code is worth having.
|
||||
|
||||
## Consequences
|
||||
|
||||
* One definition of "the suite passed".
|
||||
* `run.sh` still works standalone and unchanged for local use and for
|
||||
`run.sh --uds … --ws-port …` against a real daemon; that path is out of scope for this
|
||||
ADR, which is only about what CI treats as authoritative.
|
||||
* If PROTO ever wants the two labelled tests separated (e.g. a `conformance-e2e` label so
|
||||
jobs can select them independently without a name regex), that is a `tests/conformance/`
|
||||
change on their side; the `-E '^conformance$'` here is the PKG/QA-side stopgap until
|
||||
then.
|
||||
@@ -14,7 +14,7 @@ Do not write feature code in any lane's directory.
|
||||
1. **Toolchain script** — `tools/bootstrap.sh` installing the `apt` list in the README,
|
||||
verified on a clean 26.04 VM. The dev machine already has git 2.53, CMake 4.2.3,
|
||||
g++ 15.2, ninja, Qt 6 dev, libcurl, SQLite, nlohmann-json and ffmpeg; only
|
||||
`libqt6svg6-dev`, `libsecret-1-dev`, `nodejs`/`npm` and (optionally) `clang` are
|
||||
`qt6-svg-dev`, `libsecret-1-dev`, `nodejs`/`npm` and (optionally) `clang` are
|
||||
missing. The script must still install the full list for clean machines.
|
||||
⚠ CMake 4.x rejects `cmake_minimum_required` below 3.5 — check every dependency.
|
||||
2. **CMake** — top-level `CMakeLists.txt` + `CMakePresets.json` with presets:
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
# tests/integration — cross-lane runtime gates
|
||||
|
||||
Owned by PKG/QA. Real binaries against `tools/mockd` / `tools/testserver`, headless.
|
||||
Unit tests live in each lane; this tree is for behaviour that only shows up when the
|
||||
pieces run together (throughput, memory over time, reconnect).
|
||||
|
||||
## GUI M1 definition-of-done gates (R3)
|
||||
|
||||
`gui/docs/pkg-qa-requests-m1.md` R3: three GUI DoD items are not unit tests and have
|
||||
nowhere to run. GUI owns the harness; PKG/QA owns the CI job. This is the wiring contract
|
||||
so the two halves meet without another round trip.
|
||||
|
||||
### What PKG/QA needs from GUI
|
||||
|
||||
A driver invoked as `gui/tests/dod/run.sh <gate> [--json <path>]` (exact path TBD by GUI),
|
||||
headless-capable (Xvfb or offscreen `QT_QPA_PLATFORM`), with:
|
||||
|
||||
| `<gate>` | Pass / fail condition | Budget |
|
||||
|---|---|---|
|
||||
| `scroll-60fps` | `mockd --tasks 10000`, scripted fling scroll; **fail** if p99 frame > 16.6 ms | per-PR |
|
||||
| `rss-flat` | `mockd --tasks 10000` + progress events, 10 min; **fail** if RSS growth > a fixed slack (GUI picks the number, states it) | nightly |
|
||||
| `unhappy-path` | `mockd --slow` / `--flaky <f>` / `--drop-connection <s>`; **fail** on crash, on watchdog-detected hang, or if connection state never returns to `Connected` | per-PR |
|
||||
|
||||
Contract:
|
||||
|
||||
* exit `0` pass, non-zero fail; a hang is the harness's own watchdog to catch and turn
|
||||
into a non-zero exit, not something CI should have to `timeout(1)` around.
|
||||
* `--json` writes one machine-readable result file (measured p99, RSS series, recovery
|
||||
time) so the job can upload it as an artifact and a regression is a diff, not a re-run.
|
||||
* no network, no writes outside a tempdir, no leaked child processes on failure.
|
||||
|
||||
### CI job — pre-drafted, add once the harness path is fixed
|
||||
|
||||
```yaml
|
||||
gui-dod:
|
||||
# Per-PR GUI gates. The 10-minute rss-flat gate is in gui-dod-nightly, not here.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Bootstrap toolchain
|
||||
run: sudo ./tools/bootstrap.sh
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22' # tools/mockd
|
||||
- name: Configure + build
|
||||
run: |
|
||||
cmake --preset dev
|
||||
cmake --build --preset dev --target velox-gui
|
||||
- name: Install mockd
|
||||
run: cd tools/mockd && npm ci
|
||||
- name: Xvfb + gates
|
||||
run: |
|
||||
sudo apt-get install -y --no-install-recommends xvfb
|
||||
xvfb-run -a gui/tests/dod/run.sh scroll-60fps --json scroll.json # TODO(GUI): path
|
||||
xvfb-run -a gui/tests/dod/run.sh unhappy-path --json unhappy.json
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: gui-dod-${{ github.run_id }}
|
||||
path: "*.json"
|
||||
|
||||
gui-dod-nightly:
|
||||
if: github.event_name == 'schedule'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Bootstrap toolchain
|
||||
run: sudo ./tools/bootstrap.sh
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Configure + build
|
||||
run: |
|
||||
cmake --preset dev
|
||||
cmake --build --preset dev --target velox-gui
|
||||
- run: cd tools/mockd && npm ci
|
||||
- name: RSS soak (10 min)
|
||||
run: |
|
||||
sudo apt-get install -y --no-install-recommends xvfb
|
||||
xvfb-run -a gui/tests/dod/run.sh rss-flat --json rss.json
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: gui-dod-rss-${{ github.run_id }}
|
||||
path: rss.json
|
||||
```
|
||||
|
||||
`gui-dod-nightly` needs a `schedule:` trigger added to the top of `ci.yml` (there is none
|
||||
today), or its own workflow file. Decide that when the harness lands.
|
||||
|
||||
### Status
|
||||
|
||||
Blocked on GUI's harness. Not urgent (GUI M1 DoD, not M0). When GUI files the follow-up
|
||||
with the real `run.sh` path and the `rss-flat` slack number, PKG/QA drops the `TODO(GUI)`
|
||||
markers, adds the `schedule:` trigger, and marks `gui-dod` required.
|
||||
+33
-1
@@ -84,7 +84,9 @@ APT_GUI=(
|
||||
qt6-base-dev
|
||||
qt6-tools-dev
|
||||
qt6-tools-dev-tools # lrelease/lupdate for i18n
|
||||
libqt6svg6-dev # GUI: SVG icon rendering
|
||||
qt6-svg-dev # GUI: SVG icon rendering. Was libqt6svg6-dev, which has no
|
||||
# candidate on 26.04; qt6-svg-dev is the package that
|
||||
# ships the headers + Qt6SvgConfig.cmake find_package needs.
|
||||
)
|
||||
|
||||
APT_LINT=(
|
||||
@@ -144,6 +146,36 @@ need_cmd() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Every apt name this script would install must resolve to an installable candidate on
|
||||
# THIS release. The checks below verify outcomes — binaries on $PATH, pkg-config modules —
|
||||
# on a box that already installed everything; none of them look at the APT_* names, which
|
||||
# is the one part of the script that breaks on a clean machine of the wrong release
|
||||
# (libqt6svg6-dev did exactly this on 26.04). See gui/docs/pkg-qa-requests-m1.md R2.
|
||||
# `apt-cache policy` needs no root and no network; it reads the lists apt already has.
|
||||
log "validating apt package names (${#PKGS[@]})"
|
||||
missing_pkgs=()
|
||||
for pkg in "${PKGS[@]}"; do
|
||||
# `apt-cache policy` exits 0 whether or not the name resolves; a name with no
|
||||
# installable version prints "Candidate: (none)". Grep the line, don't trust $?.
|
||||
cand="$(apt-cache policy "$pkg" 2>/dev/null | sed -n 's/^ Candidate: //p')"
|
||||
if [[ -z "$cand" || "$cand" == "(none)" ]]; then
|
||||
missing_pkgs+=("$pkg")
|
||||
fi
|
||||
done
|
||||
if (( ${#missing_pkgs[@]} == 0 )); then
|
||||
printf ' \033[32m✓\033[0m %-16s all %d resolve on %s\n' \
|
||||
"apt names" "${#PKGS[@]}" "${VERSION_ID:-this release}"
|
||||
elif (( ${#missing_pkgs[@]} == ${#PKGS[@]} )); then
|
||||
warn "no apt candidate for ANY of the ${#PKGS[@]} packages — the apt lists are almost"
|
||||
warn "certainly stale, not the package set. Run 'apt-get update' and re-run --check."
|
||||
else
|
||||
for pkg in "${missing_pkgs[@]}"; do
|
||||
printf ' \033[31m✗\033[0m %-16s no installable candidate on %s\n' \
|
||||
"$pkg" "${VERSION_ID:-this release}"
|
||||
done
|
||||
fail=1
|
||||
fi
|
||||
|
||||
need_cmd git git --version
|
||||
need_cmd cmake cmake --version
|
||||
need_cmd ninja ninja --version
|
||||
|
||||
Reference in New Issue
Block a user