/** * Conformance runner, TypeScript side. * * Replays every fixture in contracts/fixtures against a live server — mockd today, veloxd * from M1 — through the generated client types and validators. The same suite runs against * both, which is the point: if a lane drifts from the contract, this goes red the same day * rather than at M2 integration. * * What each fixture asserts * success the reply carries a result; the result passes the generated validator; its * shape matches the golden file * error the reply carries an error with the fixture's code * timeout nothing arrives inside the deadline, and the client is expected to give up. * This is capture.offer's fail-open guarantee, and it is a pass when the * server stays silent. * * Values are compared by *shape*, not by equality: a live daemon returns its own task ids * and its own clock, and demanding byte-identical results would only teach the suite to * lie. Types, key sets and error codes are compared exactly. * * npx tsx replay.ts --uds /run/user/1000/velox/velox.sock --ws-port 52000 */ import { readFileSync, readdirSync, statSync } from 'node:fs'; import { join, relative, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { connectUds, connectWs, type Conn, type TransportName } from './client.js'; import { METHODS, isMethodName, type MethodName, } from '../../../extension/src/shared/protocol/methods.js'; import { isEventName } from '../../../extension/src/shared/protocol/events.js'; import { validateEventParams, validateParams, validateResult, } from '../../../extension/src/shared/protocol/validate.js'; const HERE = resolve(fileURLToPath(import.meta.url), '..'); const REPO = resolve(HERE, '..', '..', '..'); const FIXTURES = resolve(REPO, 'contracts', 'fixtures'); const PLACEHOLDERS = new Set(['$uuid', '$isoDate', '$any', '$opaque', '$taskId', '$taskId2']); /** * Concrete stand-ins for the placeholders, used when a golden payload is validated on its * own. A validator applies length and pattern rules, so "$opaque" has to become something * token-shaped before it is checked. */ const CONCRETE: Record = { $uuid: 'e6f0a1b2-3c4d-4e5f-8a9b-0c1d2e3f4a5b', $taskId: 'e6f0a1b2-3c4d-4e5f-8a9b-0c1d2e3f4a5b', $taskId2: '11112222-3333-4444-8555-666677778888', $isoDate: '2026-09-09T10:14:52Z', $any: 'placeholder', $opaque: 'cGxhY2Vob2xkZXItdG9rZW4tNjQtYnl0ZXMtb2YtZW50cm9weS1nb2VzLWhlcmU', }; function concrete(value: unknown): unknown { if (typeof value === 'string') return CONCRETE[value] ?? value; if (Array.isArray(value)) return value.map(concrete); if (value && typeof value === 'object') { const out: Record = {}; for (const [k, v] of Object.entries(value as Record)) out[k] = concrete(v); return out; } return value; } interface Fixture { file: string; name: string; kind?: 'timeout'; /** A condition the server cannot produce from the request alone. Skipped unless the * harness has arranged it — see tests/integration. */ requires?: string; transport?: TransportName; deadlineMs?: number; request?: { jsonrpc: '2.0'; id: number | string; method: string; params?: unknown }; notification?: { jsonrpc: '2.0'; method: string; params: unknown }; response?: { jsonrpc: '2.0'; id: number | string; result?: unknown; error?: { code: number } } | null; } interface Outcome { fixture: string; transport: TransportName | 'static'; ok: boolean; detail: string; } // ---------------------------------------------------------------- shape match /** * Compare an actual value against a golden one structurally. Placeholders match anything; * objects must have the same keys; arrays must agree on emptiness and on element shape. */ function shapeMismatch(golden: unknown, actual: unknown, path = ''): string | null { if (typeof golden === 'string' && PLACEHOLDERS.has(golden)) return null; // The generated validator has already ruled on whether null is allowed here, so a null // is never a shape failure: a golden file shows one plausible value, not the only one. if (actual === null) return null; if (golden === null) return actual === null ? null : `${path}: expected null, got ${typeName(actual)}`; if (Array.isArray(golden)) { if (!Array.isArray(actual)) return `${path}: expected an array, got ${typeName(actual)}`; if (golden.length > 0 && actual.length === 0) return `${path}: expected a non-empty array`; if (golden.length > 0 && actual.length > 0) return shapeMismatch(golden[0], actual[0], `${path}/0`); return null; } if (typeof golden === 'object') { if (typeof actual !== 'object' || actual === null || Array.isArray(actual)) return `${path}: expected an object, got ${typeName(actual)}`; const g = golden as Record; const a = actual as Record; for (const key of Object.keys(g)) { // A golden null means "may be absent"; the contract treats absent and null alike. if (!(key in a)) { if (g[key] === null) continue; return `${path}/${key}: missing from the response`; } const sub = shapeMismatch(g[key], a[key], `${path}/${key}`); if (sub) return sub; } for (const key of Object.keys(a)) { if (!(key in g)) return `${path}/${key}: not in the contract's result`; } return null; } if (typeof golden !== typeof actual) return `${path}: expected ${typeof golden}, got ${typeName(actual)}`; return null; } function typeName(v: unknown): string { if (v === null) return 'null'; if (Array.isArray(v)) return 'array'; return typeof v; } // ------------------------------------------------------------------- fixtures function walk(dir: string): string[] { const out: string[] = []; for (const entry of readdirSync(dir)) { const full = join(dir, entry); if (statSync(full).isDirectory()) out.push(...walk(full)); else if (entry.endsWith('.json')) out.push(full); } return out; } function loadFixtures(): Fixture[] { return walk(FIXTURES).map((file) => ({ ...(JSON.parse(readFileSync(file, 'utf8')) as Omit), file: relative(REPO, file), })); } // -------------------------------------------------------------------- checks /** Runs with no server: the generated validators must accept every golden payload. */ function staticChecks(fixtures: readonly Fixture[]): Outcome[] { const out: Outcome[] = []; for (const f of fixtures) { if (f.notification) { const name = f.notification.method; if (!isEventName(name)) { out.push({ fixture: f.file, transport: 'static', ok: false, detail: `unknown event ${name}` }); continue; } const r = validateEventParams(name, concrete(f.notification.params)); out.push({ fixture: f.file, transport: 'static', ok: r.ok, detail: r.ok ? 'event payload validates' : `${r.path}: ${r.message}` }); continue; } const method = f.request?.method; if (method === undefined || !isMethodName(method)) continue; const expectsInvalidParams = f.response?.error?.code === -32602; const r = validateParams(method, concrete(f.request?.params ?? {})); if (expectsInvalidParams) { out.push({ fixture: f.file, transport: 'static', ok: !r.ok, detail: r.ok ? 'expects -32602 but the params validate' : 'params correctly rejected' }); } else { out.push({ fixture: f.file, transport: 'static', ok: r.ok, detail: r.ok ? 'params validate' : `${r.path}: ${r.message}` }); } if (f.response && 'result' in f.response) { const rr = validateResult(method, concrete(f.response.result)); out.push({ fixture: f.file, transport: 'static', ok: rr.ok, detail: rr.ok ? 'golden result validates' : `${rr.path}: ${rr.message}` }); } } return out; } /** * Methods that destroy the state later fixtures rely on. Replayed last so the suite does * not depend on file order, which is the sort of thing that goes green locally and red in * CI on a different filesystem. */ const DESTRUCTIVE = new Set(['download.remove']); function replayOrder(a: Fixture, b: Fixture): number { const rank = (f: Fixture): number => (DESTRUCTIVE.has(f.request?.method ?? '') ? 1 : 0); return rank(a) - rank(b) || a.file.localeCompare(b.file); } /** Substitute the ids the runner bound during setup into a fixture's params. */ function bind(value: unknown, bindings: Record): unknown { if (typeof value === 'string') return bindings[value] ?? value; if (Array.isArray(value)) return value.map((v) => bind(v, bindings)); if (value && typeof value === 'object') { const out: Record = {}; for (const [k, v] of Object.entries(value as Record)) out[k] = bind(v, bindings); return out; } return value; } /** * Create the tasks the task-referencing fixtures bind to. Doing this per connection is * what lets the same suite run against an empty veloxd and against a seeded mockd. * * A server that cannot even complete this setup (not a fixture, so nothing above can * report it) still needs to be visible as a failure rather than an uncaught exception * that takes the whole runner down before a single fixture is checked — so a failure * here becomes an Outcome and setup moves on, leaving that binding unresolved (its * fixtures will then fail on the placeholder, individually, same as any other bad value). */ async function setupBindings(conn: Conn): Promise<{ bindings: Record; setup: Outcome[] }> { const bindings: Record = {}; const setup: Outcome[] = []; for (const [key, url] of [['$taskId', 'https://example.org/conformance-a.bin'], ['$taskId2', 'https://example.org/conformance-b.bin']] as const) { try { const added = await conn.call('download.add', { url, startMode: 'later' }); bindings[key] = added.taskId; setup.push({ fixture: `setup/${key}`, transport: conn.transport, ok: true, detail: 'download.add for the binding succeeded' }); } catch (err) { setup.push({ fixture: `setup/${key}`, transport: conn.transport, ok: false, detail: `download.add for the binding failed: ${String(err)}` }); } } return { bindings, setup }; } async function replay(conn: Conn, fixtures: readonly Fixture[], bindings: Record, includeRequires = false): Promise { const out: Outcome[] = []; const t = conn.transport; for (const f of [...fixtures].sort(replayOrder)) { if (!f.request) continue; const method = f.request.method; if (f.transport !== undefined && f.transport !== t) continue; if (!isMethodName(method)) continue; if (!(METHODS[method].transports as readonly string[]).includes(t)) continue; if (f.requires !== undefined && !includeRequires) { out.push({ fixture: f.file, transport: t, ok: true, detail: `skipped: requires ${f.requires}` }); continue; } const deadline = f.deadlineMs ?? Math.max(METHODS[method].deadlineMs, 2000); const frame = await conn.request(method, bind(f.request.params ?? {}, bindings), deadline); if (f.kind === 'timeout') { out.push({ fixture: f.file, transport: t, ok: frame === null, detail: frame === null ? `no reply within ${deadline} ms — the client fails open, as it must` : 'the server answered a fixture that requires silence', }); continue; } if (frame === null) { out.push({ fixture: f.file, transport: t, ok: false, detail: `no reply within ${deadline} ms` }); continue; } const expected = f.response; if (expected && 'error' in expected && expected.error) { const got = frame.error?.code; out.push({ fixture: f.file, transport: t, ok: got === expected.error.code, detail: got === expected.error.code ? `error ${got} as documented` : `expected error ${expected.error.code}, got ${frame.error ? `error ${got}` : 'a result'}`, }); continue; } if (frame.error) { out.push({ fixture: f.file, transport: t, ok: false, detail: `expected a result, got error ${frame.error.code}: ${frame.error.message}` }); continue; } const validated = validateResult(method, frame.result); if (!validated.ok) { out.push({ fixture: f.file, transport: t, ok: false, detail: `result fails the generated validator at ${validated.path}: ${validated.message}` }); continue; } const mismatch = expected && 'result' in expected ? shapeMismatch(expected.result, frame.result) : null; out.push({ fixture: f.file, transport: t, ok: mismatch === null, detail: mismatch ?? 'result validates and matches the golden shape' }); } return out; } /** The transport rules are part of the contract, so they get replayed too. */ async function privilegeChecks(conn: Conn): Promise { if (conn.transport !== 'ws') return []; const out: Outcome[] = []; const privileged = (Object.keys(METHODS) as MethodName[]).filter((m) => METHODS[m].privileged); for (const method of privileged) { const frame = await conn.request(method, {}, 3000); const ok = frame?.error?.code === -32003; out.push({ fixture: `transport-rules/${method}`, transport: 'ws', ok, detail: ok ? 'refused with -32003 over the WebSocket, as required' : `expected -32003, got ${frame ? JSON.stringify(frame.error ?? frame.result).slice(0, 80) : 'no reply'}`, }); } return out; } // ----------------------------------------------------------- expected failure /** * A fixture this runner is allowed to fail against the target server, with why. Used * against veloxd, which still has stub handlers (daemon/docs/deferrals.md D1-D4b) that * mockd does not: mockd always answers every fixture correctly, so this list is empty * there and the mechanism does not apply. * * `fixture` matches Outcome.fixture exactly (the path printed in a FAIL line, e.g. * "contracts/fixtures/download.pause.json"); `transport`, if given, narrows to one * transport. This is a maintained allowlist, not a captured snapshot: an entry that no * longer fails is a bug in the list, not a pass, so `applyXfail` turns that back into a * failure rather than silently dropping the entry. That is what keeps the list shrinking * as DAEMON lands handlers instead of quietly becoming a list nobody rechecks. */ interface XfailEntry { fixture: string; transport?: TransportName; reason: string; } function loadXfail(path: string): XfailEntry[] { const parsed = JSON.parse(readFileSync(path, 'utf8')) as unknown; if (!Array.isArray(parsed)) throw new Error(`${path}: expected a JSON array`); return parsed as XfailEntry[]; } /** * Reconciles outcomes against the allowlist. A listed fixture that failed is downgraded * to a pass (its detail says why). A listed fixture that *passed* is flipped to a * failure: the entry is stale and must be deleted from the list, not left to rot. */ function applyXfail(results: readonly Outcome[], xfail: readonly XfailEntry[]): Outcome[] { const matches = (e: XfailEntry, r: Outcome): boolean => e.fixture === r.fixture && (e.transport === undefined || e.transport === r.transport); return results.map((r) => { const entry = xfail.find((e) => matches(e, r)); if (!entry) return r; if (!r.ok) { return { ...r, ok: true, detail: `xfail (${entry.reason}): ${r.detail}` }; } return { ...r, ok: false, detail: `xfail entry unexpectedly passed — delete it from the allowlist ` + `(was: ${entry.reason})`, }; }); } // ---------------------------------------------------------------------- main async function main(): Promise { const argv = process.argv.slice(2); const arg = (name: string): string | undefined => { const i = argv.indexOf(name); return i === -1 ? undefined : argv[i + 1]; }; // --only narrows the run to fixtures whose path contains a substring, and // --include-requires replays the ones needing a condition the harness has arranged // (a slow daemon, a hostile origin server). run.sh uses both to prove capture.offer // fails open, which cannot be shown against a healthy server. const only = arg('--only'); const includeRequires = argv.includes('--include-requires'); const all = loadFixtures(); const fixtures = only === undefined ? all : all.filter((f) => f.file.includes(only)); if (fixtures.length === 0) { process.stderr.write(`conformance: --only ${String(only)} matched no fixtures\n`); process.exit(2); } const results: Outcome[] = [...staticChecks(fixtures)]; const udsPath = arg('--uds'); const wsPort = arg('--ws-port'); if (udsPath) { const conn = await connectUds(udsPath); await conn.call('session.hello', { clientType: 'test', clientName: 'conformance', protocolVersion: '1.0.0' }); const { bindings, setup } = await setupBindings(conn); results.push(...setup, ...(await replay(conn, fixtures, bindings, includeRequires))); conn.close(); } if (wsPort) { const conn = await connectWs(Number(wsPort)); const paired = await conn.request( 'session.pair', { clientName: 'conformance', extensionId: '11111111-2222-3333-4444-555555555555' }, 5000, ); const token = (paired?.result as { token?: string } | undefined)?.token; if (token === undefined) throw new Error('pairing failed: no token issued'); await conn.request('session.hello', { clientType: 'test', clientName: 'conformance', protocolVersion: '1.0.0', token }, 5000); const { bindings, setup } = await setupBindings(conn); results.push(...setup, ...(await replay(conn, fixtures, bindings, includeRequires))); results.push(...(await privilegeChecks(conn))); conn.close(); } if (!udsPath && !wsPort) { process.stdout.write('no --uds or --ws-port given: ran static checks only\n'); } const xfailPath = arg('--xfail'); const finalResults = xfailPath ? applyXfail(results, loadXfail(xfailPath)) : results; const failed = finalResults.filter((r) => !r.ok); for (const r of failed) { process.stdout.write(`FAIL [${r.transport}] ${r.fixture}\n ${r.detail}\n`); } const byTransport = new Map(); for (const r of finalResults) byTransport.set(r.transport, (byTransport.get(r.transport) ?? 0) + 1); const summary = [...byTransport].map(([k, v]) => `${k}:${v}`).join(' '); process.stdout.write( `\n${finalResults.length - failed.length}/${finalResults.length} checks passed (${summary})\n`); process.exit(failed.length === 0 ? 0 : 1); } main().catch((err: unknown) => { process.stderr.write(`conformance: ${String(err)}\n`); process.exit(2); });