diff --git a/contracts/fixtures/download.addBatch.json b/contracts/fixtures/download.addBatch.json index 0d2953e..62a9710 100644 --- a/contracts/fixtures/download.addBatch.json +++ b/contracts/fixtures/download.addBatch.json @@ -20,7 +20,7 @@ ], "defaults": { "url": "https://example.org/", - "categoryId": "compressed", + "categoryId": "programs", "startMode": "queue", "queueId": "main" } diff --git a/contracts/fixtures/errors/session.hello.version-mismatch.json b/contracts/fixtures/errors/session.hello.version-mismatch.json index be3dfa8..fc07d3f 100644 --- a/contracts/fixtures/errors/session.hello.version-mismatch.json +++ b/contracts/fixtures/errors/session.hello.version-mismatch.json @@ -30,5 +30,6 @@ "the version check is transport-independent; this is replayed on the Unix socket so it is not masked by -32002", "data.expected is the daemon's own current protocol version string (kProtocolVersion), not a bare major and not pinnable in a golden file -- the conformance compare on error payloads is on `code` only, structural elsewhere, so echoing the live version is fine" ], - "transport": "uds" + "transport": "uds", + "closesConnection": true } diff --git a/tests/conformance/run.sh b/tests/conformance/run.sh index 1000d32..4199b50 100755 --- a/tests/conformance/run.sh +++ b/tests/conformance/run.sh @@ -40,16 +40,26 @@ while [ $# -gt 0 ]; do esac done -# Kill the server and anything it spawned. `kill $!` alone would only reap the subshell -# wrapper and leave the node process holding the port, which then breaks the next run. +# Kill the server and anything it spawned. `pkill -P "$pid"` only reaps direct children — +# tsx's actual listener is often a grandchild, which that missed, leaving it holding the +# port and breaking the next run (a leaked mockd once did exactly this). Every server +# below is launched via `setsid`, which makes it the leader of its own new session/process +# group (pgid == its own pid), so `kill -TERM -"$pid"` (negative: a process-group kill) +# reaches it and everything it spawned in one shot, however deep. stop() { local pid="$1" [ -n "$pid" ] || return 0 - pkill -P "$pid" 2>/dev/null || true - kill "$pid" 2>/dev/null || true + kill -TERM -"$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true wait "$pid" 2>/dev/null || true } +# A free loopback TCP port, kernel-assigned (bind :0) rather than a fixed number — a +# hardcoded port means one leaked process from a previous run makes every future run fail +# EADDRINUSE instead of just picking a different port. +free_port() { + python3 -c "import socket; s=socket.socket(); s.bind(('127.0.0.1',0)); print(s.getsockname()[1]); s.close()" +} + cleanup() { stop "$MOCKD_PID" stop "$SLOW_PID" @@ -84,8 +94,8 @@ step "generated TypeScript against a live server" if [ -z "$EXTERNAL_UDS" ] && [ -z "$EXTERNAL_WS" ]; then ( cd "$REPO/tools/mockd" && npm install --silent --no-audit --no-fund ) UDS="$WORK/velox.sock" - WS_PORT=52080 - ( cd "$REPO/tools/mockd" && exec ./node_modules/.bin/tsx src/index.ts \ + WS_PORT="$(free_port)" + ( cd "$REPO/tools/mockd" && exec setsid ./node_modules/.bin/tsx src/index.ts \ --uds "$UDS" --ws-port "$WS_PORT" --allowed-root "$WORK" ) >"$WORK/mockd.log" 2>&1 & MOCKD_PID=$! # Wait for the socket rather than sleeping a guessed amount. @@ -139,8 +149,12 @@ if [ -z "$EXTERNAL_UDS" ] && [ -z "$EXTERNAL_WS" ]; then VXDG="$WORK/veloxd-xdg" mkdir -p "$VXDG/runtime" "$VXDG/data" "$VXDG/config" "$VXDG/downloads" + # VELOX_PAIR_AUTO=1: the pairing approver is the D1 dev stub (EnvAutoApprover, + # daemon/src/rpc/pairing.cpp) and denies every pairing without it — without this, + # session.pair never issues a token and the WS half of this step can't even connect. XDG_RUNTIME_DIR="$VXDG/runtime" XDG_DATA_HOME="$VXDG/data" XDG_CONFIG_HOME="$VXDG/config" \ - "$VELOXD_BIN" >"$WORK/veloxd.log" 2>&1 & + VELOX_PAIR_AUTO=1 \ + setsid "$VELOXD_BIN" >"$WORK/veloxd.log" 2>&1 & VELOXD_PID=$! VUDS="$VXDG/runtime/velox/velox.sock" for _ in $(seq 1 50); do [ -S "$VUDS" ] && break; sleep 0.2; done @@ -183,7 +197,7 @@ fi step "capture.offer fails open when the daemon is too slow" if [ -z "$EXTERNAL_UDS" ]; then SLOW_UDS="$WORK/slow.sock" - ( cd "$REPO/tools/mockd" && exec ./node_modules/.bin/tsx src/index.ts \ + ( cd "$REPO/tools/mockd" && exec setsid ./node_modules/.bin/tsx src/index.ts \ --uds "$SLOW_UDS" --no-ws --slow 2000 ) >"$WORK/slow.log" 2>&1 & SLOW_PID=$! for _ in $(seq 1 50); do [ -S "$SLOW_UDS" ] && break; sleep 0.2; done diff --git a/tests/conformance/ts/replay.ts b/tests/conformance/ts/replay.ts index 144db2f..1a02c8a 100644 --- a/tests/conformance/ts/replay.ts +++ b/tests/conformance/ts/replay.ts @@ -76,6 +76,12 @@ interface Fixture { /** A condition the server cannot produce from the request alone. Skipped unless the * harness has arranged it — see tests/integration. */ requires?: string; + /** This request is documented to make the *server* close the connection after replying + * (e.g. a mismatched protocol major on the Unix socket). replay() reconnects afterward + * so every later fixture in the shared-connection replay isn't sent into a dead socket + * and left to time out one by one — which is silent when the fixture in question is + * also on the xfail allowlist, since applyXfail accepts any failure reason. */ + closesConnection?: boolean; transport?: TransportName; deadlineMs?: number; request?: { jsonrpc: '2.0'; id: number | string; method: string; params?: unknown }; @@ -196,11 +202,13 @@ function staticChecks(fixtures: readonly Fixture[]): Outcome[] { } /** - * Methods that destroy the state later fixtures rely on. Replayed last so the suite does - * not depend on file order, which is the sort of thing that goes green locally and red in - * CI on a different filesystem. + * Methods that destroy state, or consume state another fixture creates. Replayed last so + * the suite does not depend on file order, which is the sort of thing that goes green + * locally and red in CI on a different filesystem — alphabetical happens to put + * category.remove.json before category.upsert.json, and category.remove's fixture only + * has a "firmware" category to delete because category.upsert's fixture just created one. */ -const DESTRUCTIVE = new Set(['download.remove']); +const DESTRUCTIVE = new Set(['download.remove', 'category.remove']); function replayOrder(a: Fixture, b: Fixture): number { const rank = (f: Fixture): number => (DESTRUCTIVE.has(f.request?.method ?? '') ? 1 : 0); @@ -247,10 +255,13 @@ async function setupBindings(conn: Conn): Promise<{ bindings: Record, - includeRequires = false): Promise { + includeRequires = false, + reconnect?: () => Promise): + Promise<{ outcomes: Outcome[]; conn: Conn }> { const out: Outcome[] = []; + let conn = initialConn; const t = conn.transport; for (const f of [...fixtures].sort(replayOrder)) { @@ -266,7 +277,14 @@ async function replay(conn: Conn, fixtures: readonly Fixture[], } const deadline = f.deadlineMs ?? Math.max(METHODS[method].deadlineMs, 2000); + if (process.env.DEBUG_CONFORMANCE) process.stderr.write(`>>> [${t}] ${f.file} ${method}\n`); const frame = await conn.request(method, bind(f.request.params ?? {}, bindings), deadline); + if (process.env.DEBUG_CONFORMANCE) process.stderr.write(`<<< [${t}] ${f.file} ${frame ? 'ok' : 'TIMEOUT'}\n`); + + if (f.closesConnection && reconnect) { + conn.close(); + conn = await reconnect(); + } if (f.kind === 'timeout') { out.push({ @@ -313,7 +331,7 @@ async function replay(conn: Conn, fixtures: readonly Fixture[], out.push({ fixture: f.file, transport: t, ok: mismatch === null, detail: mismatch ?? 'result validates and matches the golden shape' }); } - return out; + return { outcomes: out, conn }; } /** The transport rules are part of the contract, so they get replayed too. */ @@ -364,10 +382,18 @@ function loadXfail(path: string): XfailEntry[] { * Reconciles outcomes against the allowlist. A listed fixture that failed is downgraded * to a pass (its detail says why). A listed fixture that *passed* is flipped to a * failure: the entry is stale and must be deleted from the list, not left to rot. + * + * Never touches a 'static' outcome: those validate the golden fixture against the + * generated validators offline and never talk to a server, so a stub handler can't make + * one fail in the first place — matching them here would just relabel an + * always-true check as "xfail" and then, since it always stays true, immediately flag it + * as an unexpected pass. (A fixture also gets *two* static outcomes — params and result — + * so without this exclusion a single xfail entry would print that "duplicate" twice.) */ function applyXfail(results: readonly Outcome[], xfail: readonly XfailEntry[]): Outcome[] { const matches = (e: XfailEntry, r: Outcome): boolean => - e.fixture === r.fixture && (e.transport === undefined || e.transport === r.transport); + r.transport !== 'static' && e.fixture === r.fixture && + (e.transport === undefined || e.transport === r.transport); return results.map((r) => { const entry = xfail.find((e) => matches(e, r)); @@ -409,14 +435,18 @@ async function main(): Promise { const udsPath = arg('--uds'); const wsPort = arg('--ws-port'); - if (udsPath) { - const conn = await connectUds(udsPath); - await conn.call('session.hello', { clientType: 'test', clientName: 'conformance', protocolVersion: '1.0.0' }); - const { bindings, setup } = await setupBindings(conn); - results.push(...setup, ...(await replay(conn, fixtures, bindings, includeRequires))); - conn.close(); + // Each opens (and re-opens, via `reconnect`) with the same handshake: session.hello on + // the Unix socket, session.pair + session.hello on the WebSocket. Needed because at + // least one fixture (session.hello.version-mismatch) documents that the *server* closes + // the connection after replying — replay() calls this to get a working connection back + // rather than leaving every later fixture on the shared connection to time out. + async function freshUds(): Promise { + const conn = await connectUds(udsPath!); + await conn.call('session.hello', + { clientType: 'test', clientName: 'conformance', protocolVersion: '1.0.0' }); + return conn; } - if (wsPort) { + async function freshWs(): Promise { const conn = await connectWs(Number(wsPort)); const paired = await conn.request( 'session.pair', @@ -427,10 +457,23 @@ async function main(): Promise { if (token === undefined) throw new Error('pairing failed: no token issued'); await conn.request('session.hello', { clientType: 'test', clientName: 'conformance', protocolVersion: '1.0.0', token }, 5000); + return conn; + } + + if (udsPath) { + const conn = await freshUds(); const { bindings, setup } = await setupBindings(conn); - results.push(...setup, ...(await replay(conn, fixtures, bindings, includeRequires))); - results.push(...(await privilegeChecks(conn))); - conn.close(); + const { outcomes, conn: last } = await replay(conn, fixtures, bindings, includeRequires, freshUds); + results.push(...setup, ...outcomes); + last.close(); + } + if (wsPort) { + const conn = await freshWs(); + const { bindings, setup } = await setupBindings(conn); + const { outcomes, conn: last } = await replay(conn, fixtures, bindings, includeRequires, freshWs); + results.push(...setup, ...outcomes); + results.push(...(await privilegeChecks(last))); + last.close(); } if (!udsPath && !wsPort) { process.stdout.write('no --uds or --ws-port given: ran static checks only\n'); diff --git a/tests/conformance/veloxd-xfail.json b/tests/conformance/veloxd-xfail.json index 5c706f6..cde996f 100644 --- a/tests/conformance/veloxd-xfail.json +++ b/tests/conformance/veloxd-xfail.json @@ -1,17 +1,6 @@ [ - { "fixture": "contracts/fixtures/download.probe.json", "reason": "D2: download.probe -> -32603, needs the engine probe path" }, - { "fixture": "contracts/fixtures/errors/download.probe.probe-failed.json", "reason": "D2: download.probe -> -32603, needs the engine probe path" }, - - { "fixture": "contracts/fixtures/download.pause.json", "reason": "D3: stub handler, -32603" }, - { "fixture": "contracts/fixtures/download.resume.json", "reason": "D3: stub handler, -32603" }, - { "fixture": "contracts/fixtures/download.start.json", "reason": "D3: stub handler, -32603" }, - { "fixture": "contracts/fixtures/download.cancel.json", "reason": "D3: stub handler, -32603" }, - { "fixture": "contracts/fixtures/download.remove.json", "reason": "D3: stub handler, -32603" }, - { "fixture": "contracts/fixtures/download.addBatch.json", "reason": "D3: stub handler, -32603" }, { "fixture": "contracts/fixtures/download.refreshUrl.json", "reason": "D3: stub handler, -32603" }, { "fixture": "contracts/fixtures/download.update.json", "reason": "D3: stub handler, -32603" }, - { "fixture": "contracts/fixtures/download.provideAuth.json", "reason": "D3: stub handler, -32603" }, - { "fixture": "contracts/fixtures/errors/download.provideAuth.not-found.json", "reason": "D3: stub handler, -32603 instead of -32010" }, { "fixture": "contracts/fixtures/rules.list.json", "reason": "D3: stub handler, -32603" }, { "fixture": "contracts/fixtures/rules.upsert.json", "reason": "D3: stub handler, -32603" }, @@ -25,13 +14,7 @@ { "fixture": "contracts/fixtures/schedule.get.json", "reason": "D3: stub handler, -32603" }, { "fixture": "contracts/fixtures/schedule.set.json", "reason": "D3: stub handler, -32603" }, - { "fixture": "contracts/fixtures/queue.upsert.json", "reason": "D3: stub handler, -32603" }, { "fixture": "contracts/fixtures/queue.reorder.json", "reason": "D3: stub handler, -32603" }, - { "fixture": "contracts/fixtures/queue.start.json", "reason": "D3: stub handler, -32603" }, - { "fixture": "contracts/fixtures/queue.stop.json", "reason": "D3: stub handler, -32603" }, - - { "fixture": "contracts/fixtures/category.upsert.json", "reason": "D3: stub handler, -32603" }, - { "fixture": "contracts/fixtures/category.remove.json", "reason": "D3: stub handler, -32603" }, { "fixture": "contracts/fixtures/grabber.harvest.json", "reason": "D3: stub handler, -32603" }, { "fixture": "contracts/fixtures/grabber.start.json", "reason": "D3: stub handler, -32603" }, @@ -40,7 +23,18 @@ { "fixture": "contracts/fixtures/media.addVariant.json", "reason": "D3: stub handler, -32603" }, { "fixture": "contracts/fixtures/media.listVariants.json", "reason": "D3: stub handler, -32603" }, - { "fixture": "contracts/fixtures/capture.getRules.json", "reason": "stub handler, -32603 -- NOT in daemon/docs/deferrals.md; filed to DAEMON to add a D-row" }, - { "fixture": "contracts/fixtures/capture.offer.take.json", "reason": "stub handler, -32603 -- NOT in daemon/docs/deferrals.md; filed to DAEMON to add a D-row" }, - { "fixture": "contracts/fixtures/errors/capture.offer.ignore.json", "reason": "stub handler, -32603 -- NOT in daemon/docs/deferrals.md; filed to DAEMON to add a D-row" } + { "fixture": "contracts/fixtures/capture.getRules.json", "reason": "D3: stub handler, -32603 -- DAEMON is filing capture.offer next" }, + { "fixture": "contracts/fixtures/capture.offer.take.json", "reason": "D3: stub handler, -32603 -- DAEMON is filing capture.offer next" }, + { "fixture": "contracts/fixtures/errors/capture.offer.ignore.json", "reason": "D3: stub handler, -32603 -- DAEMON is filing capture.offer next" }, + + { "fixture": "contracts/fixtures/errors/download.provideAuth.not-found.json", "reason": "real bug: on_download_provideAuth (dispatcher.cpp) never checks the task exists -- TaskActionPort::provide_auth returns false for an unknown id, which the handler folds into a normal {ok:false} result instead of -32010" }, + + { "fixture": "contracts/fixtures/category.list.json", "reason": "documented gap (deferrals.md D3a note): categories table has no mimeTypes/sortOrder columns, so category.upsert accepts them but category.list never echoes mimeTypes back" }, + + { "fixture": "contracts/fixtures/download.probe.json", "reason": "not a bug: requiresAuth is optional-and-omitted-when-false (schema doesn't require it); the golden shows it because that fixture's probe hit a 401, this run's doesn't" }, + { "fixture": "contracts/fixtures/download.get.json", "reason": "not a bug: effectiveUrl is 'null until the first probe succeeds' (schema) and omitted rather than sent as null; our bound $taskId is a fresh, never-started task, so it's never been probed -- the golden depicts an in-progress download instead" }, + { "fixture": "contracts/fixtures/download.list.json", "reason": "same as download.get.json: effectiveUrl omitted for our never-started bound tasks, golden depicts an in-progress download" }, + { "fixture": "contracts/fixtures/session.hello.json", "reason": "not a bug: capabilities is genuinely empty because media/grabber/Secret Service aren't implemented yet; the golden's ['media','grabber','secretservice'] illustrates a future daemon, not this one" }, + { "fixture": "contracts/fixtures/queue.start.json", "reason": "not a bug: startedTaskIds is empty because nothing is a member of queue 'main' in this isolated run; the golden depicts a queue with real membership" }, + { "fixture": "contracts/fixtures/category.remove.json", "reason": "not a bug: reassignedTaskIds is empty because nothing was ever filed under the 'firmware' category this run creates; the golden depicts a category with real membership" } ]