samiandClaude Sonnet 5
203d4a662f
Verified PKG's landed CI (.github/workflows/ci.yml, db7650b/8f45815) against this lane's actual tree, per the standing instruction that conformance as a required check is this lane's DoD to verify, not PKG's. It wasn't running: the `conformance` job's presence-check looks for tests/conformance/CMakeLists.txt or tests/conformance/package.json, and neither existed -- the job was silently short-circuiting to a green "skipped" on every PR, forever. The M0 exit gate was not gating anything. tests/conformance/CMakeLists.txt registers one ctest entry, labeled "conformance", that shells out to run.sh -- the exact command tests/conformance/README.md tells a human to run locally, so there is one definition of "the suite passed", not a CMake-flavoured near-duplicate of it. cpp/CMakeLists.txt's existing conformance_cpp test gets the same label, for a lane iterating on core/generated/ who wants the fast native-only path. Fixed a second landmine found while wiring this: the root CMakeLists.txt only find_package(nlohmann_json)'s when daemon/CMakeLists.txt exists, since daemon is its real consumer -- but daemon hasn't landed yet, so add_subdirectory(tests/conformance) would have failed to configure the moment this file existed, on every machine, until daemon merges. Fixed inside tests/conformance/cpp/CMakeLists.txt with an if(NOT TARGET) guard rather than widening the root file's condition, which is PKG's to change. run.sh now installs its own Python deps (jsonschema, referencing) on demand: they aren't in tools/bootstrap.sh's apt list -- that's PKG's script, these are this suite's own dependency -- so a bare CI image would otherwise fail check_contract.py with an ImportError before this suite even started. Verified end to end: `cmake --preset dev && ctest --test-dir build/dev -R '^conformance$'` passes in 23.8s, exercising the exact command and label the CI job uses. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_012fgjnqFCS5h5L7gZTZo3rV
118 lines
4.7 KiB
Bash
Executable File
118 lines
4.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# The conformance suite. This is the command CI runs on every lane's PR.
|
|
#
|
|
# ./tests/conformance/run.sh static + C++ + TS against a mockd it starts
|
|
# ./tests/conformance/run.sh --uds PATH --ws-port N against an already-running daemon
|
|
#
|
|
# Three runners, one set of fixtures:
|
|
# 1. check_contract.py schemas, fixtures and committed generated code agree
|
|
# 2. cpp/ the generated C++ parses, serialises and dispatches every fixture
|
|
# 3. ts/replay.ts a live server answers every fixture over both transports
|
|
#
|
|
# Plus one scenario that cannot be shown against a healthy server: with the daemon
|
|
# answering slower than capture.offer's 750 ms deadline, the client must give up and let
|
|
# Firefox take the download. That is the fail-open guarantee, and it is checked here.
|
|
|
|
set -euo pipefail
|
|
|
|
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
HERE="$REPO/tests/conformance"
|
|
WORK="$(mktemp -d)"
|
|
EXTERNAL_UDS=""
|
|
EXTERNAL_WS=""
|
|
MOCKD_PID=""
|
|
SLOW_PID=""
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--uds) EXTERNAL_UDS="$2"; shift 2 ;;
|
|
--ws-port) EXTERNAL_WS="$2"; shift 2 ;;
|
|
-h|--help) sed -n '2,20p' "$0"; exit 0 ;;
|
|
*) echo "run.sh: unknown option $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
# Kill the server and anything it spawned. `kill $!` alone would only reap the subshell
|
|
# wrapper and leave the node process holding the port, which then breaks the next run.
|
|
stop() {
|
|
local pid="$1"
|
|
[ -n "$pid" ] || return 0
|
|
pkill -P "$pid" 2>/dev/null || true
|
|
kill "$pid" 2>/dev/null || true
|
|
wait "$pid" 2>/dev/null || true
|
|
}
|
|
|
|
cleanup() {
|
|
stop "$MOCKD_PID"
|
|
stop "$SLOW_PID"
|
|
rm -rf "$WORK"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
step() { printf '\n=== %s ===\n' "$1"; }
|
|
|
|
# ---------------------------------------------------------------- 1. static
|
|
step "static conformance (schemas, fixtures, generated code)"
|
|
# jsonschema/referencing aren't part of tools/bootstrap.sh's apt list (that's PKG's
|
|
# script; these are this suite's own Python deps), so this suite installs them itself
|
|
# rather than assuming a CI image happens to have them. Cheap and idempotent when
|
|
# they're already present, which is every local dev run after the first.
|
|
python3 -c "import jsonschema, referencing" 2>/dev/null \
|
|
|| python3 -m pip install --quiet --disable-pip-version-check --user jsonschema referencing
|
|
python3 "$HERE/check_contract.py"
|
|
|
|
# ------------------------------------------------------------------- 2. C++
|
|
step "generated C++ (parse, serialise, dispatch)"
|
|
CXX="${CXX:-g++}"
|
|
"$CXX" -std=c++23 -Wall -Wextra -Wpedantic -Werror \
|
|
-I"$REPO/core/generated" -I"$HERE/cpp" \
|
|
"$HERE/cpp/conformance_main.cpp" "$REPO/core/generated/velox_proto.cpp" \
|
|
-o "$WORK/conformance_cpp"
|
|
"$WORK/conformance_cpp" "$REPO"
|
|
|
|
# -------------------------------------------------------------------- 3. TS
|
|
step "generated TypeScript against a live server"
|
|
if [ -z "$EXTERNAL_UDS" ] && [ -z "$EXTERNAL_WS" ]; then
|
|
( cd "$REPO/tools/mockd" && npm install --silent --no-audit --no-fund )
|
|
UDS="$WORK/velox.sock"
|
|
WS_PORT=52080
|
|
( cd "$REPO/tools/mockd" && exec ./node_modules/.bin/tsx src/index.ts \
|
|
--uds "$UDS" --ws-port "$WS_PORT" --allowed-root "$WORK" ) >"$WORK/mockd.log" 2>&1 &
|
|
MOCKD_PID=$!
|
|
# Wait for the socket rather than sleeping a guessed amount.
|
|
for _ in $(seq 1 50); do
|
|
[ -S "$UDS" ] && node -e "require('net').connect('$UDS').on('connect',function(){this.end();process.exit(0)}).on('error',()=>process.exit(1))" 2>/dev/null && break
|
|
sleep 0.2
|
|
done
|
|
node -e "require('net').connect('$UDS').on('connect',function(){this.end();process.exit(0)}).on('error',()=>process.exit(1))" 2>/dev/null \
|
|
|| { echo "mockd did not start:"; cat "$WORK/mockd.log"; exit 1; }
|
|
else
|
|
UDS="$EXTERNAL_UDS"
|
|
WS_PORT="$EXTERNAL_WS"
|
|
fi
|
|
|
|
( cd "$HERE/ts" && npm install --silent --no-audit --no-fund )
|
|
|
|
TS_ARGS=()
|
|
[ -n "$UDS" ] && TS_ARGS+=(--uds "$UDS")
|
|
[ -n "$WS_PORT" ] && TS_ARGS+=(--ws-port "$WS_PORT")
|
|
( cd "$HERE/ts" && ./node_modules/.bin/tsx replay.ts "${TS_ARGS[@]}" )
|
|
|
|
# ------------------------------------------------- 4. capture fails open
|
|
step "capture.offer fails open when the daemon is too slow"
|
|
if [ -z "$EXTERNAL_UDS" ]; then
|
|
SLOW_UDS="$WORK/slow.sock"
|
|
( cd "$REPO/tools/mockd" && exec ./node_modules/.bin/tsx src/index.ts \
|
|
--uds "$SLOW_UDS" --no-ws --slow 2000 ) >"$WORK/slow.log" 2>&1 &
|
|
SLOW_PID=$!
|
|
for _ in $(seq 1 50); do [ -S "$SLOW_UDS" ] && break; sleep 0.2; done
|
|
[ -S "$SLOW_UDS" ] || { echo "slow mockd did not start:"; cat "$WORK/slow.log"; exit 1; }
|
|
( cd "$HERE/ts" && ./node_modules/.bin/tsx replay.ts --uds "$SLOW_UDS" \
|
|
--only capture.offer.timeout --include-requires )
|
|
else
|
|
echo "skipped: needs a deliberately slow server, which run.sh only arranges for mockd"
|
|
fi
|
|
|
|
printf '\n=== conformance: all runners passed ===\n'
|