proto: freeze the wire contract at 1.0.0
Schemas for the whole v1 surface: 38 methods, 9 events, 25 named types and the
JSON-RPC envelope, with x-privileged / x-transports / x-deadlineMs / x-errors
annotations that both generators emit as data rather than prose.
Four generators over one IR (contracts/codegen/schema_ir.py), so the C++ structs,
the TypeScript types and the OpenRPC document cannot disagree about what the
contract says:
gen_cpp.py -> core/generated/velox_proto.{hpp,cpp}
gen_ts.py -> extension/src/shared/protocol/
gen_openrpc.py -> contracts/openrpc.json
gen_cpp_conformance.py -> tests/conformance/cpp/fixture_dispatcher.hpp
Inbound parsing never throws: parse<T>() returns std::expected<T, ParseError> and
nlohmann's throwing ADL from_json is deliberately not emitted. Schema constraints
(minimum, maxLength, pattern, ...) become real runtime checks in both languages —
the daemon does not trust the extension and the extension does not trust the
daemon.
59 golden fixtures: a success case per method, 12 error cases, 9 events. Replayed
by tests/conformance/ against both the generated C++ and a live server over both
transports. tools/mockd serves the same fixtures with unhappy-path flags so the
GUI and EXT lanes never wait for veloxd.
run.sh also proves capture.offer fails open: with a daemon answering slower than
750 ms the client gives up and lets Firefox take the download.
core/generated/ is libveloxproto, a separate target from libveloxcore, which
still never sees JSON — see docs/adr/0009.
Co-Authored-By: Claude Opus 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_012fgjnqFCS5h5L7gZTZo3rV
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,387 @@
|
||||
/**
|
||||
* Conformance runner, TypeScript side.
|
||||
*
|
||||
* Replays every fixture in contracts/fixtures against a live server — mockd today, veloxd
|
||||
* from M1 — through the generated client types and validators. The same suite runs against
|
||||
* both, which is the point: if a lane drifts from the contract, this goes red the same day
|
||||
* rather than at M2 integration.
|
||||
*
|
||||
* What each fixture asserts
|
||||
* success the reply carries a result; the result passes the generated validator; its
|
||||
* shape matches the golden file
|
||||
* error the reply carries an error with the fixture's code
|
||||
* timeout nothing arrives inside the deadline, and the client is expected to give up.
|
||||
* This is capture.offer's fail-open guarantee, and it is a pass when the
|
||||
* server stays silent.
|
||||
*
|
||||
* Values are compared by *shape*, not by equality: a live daemon returns its own task ids
|
||||
* and its own clock, and demanding byte-identical results would only teach the suite to
|
||||
* lie. Types, key sets and error codes are compared exactly.
|
||||
*
|
||||
* npx tsx replay.ts --uds /run/user/1000/velox/velox.sock --ws-port 52000
|
||||
*/
|
||||
|
||||
import { readFileSync, readdirSync, statSync } from 'node:fs';
|
||||
import { join, relative, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { connectUds, connectWs, type Conn, type TransportName } from './client.js';
|
||||
import {
|
||||
METHODS,
|
||||
isMethodName,
|
||||
type MethodName,
|
||||
} from '../../../extension/src/shared/protocol/methods.js';
|
||||
import { isEventName } from '../../../extension/src/shared/protocol/events.js';
|
||||
import {
|
||||
validateEventParams,
|
||||
validateParams,
|
||||
validateResult,
|
||||
} from '../../../extension/src/shared/protocol/validate.js';
|
||||
|
||||
const HERE = resolve(fileURLToPath(import.meta.url), '..');
|
||||
const REPO = resolve(HERE, '..', '..', '..');
|
||||
const FIXTURES = resolve(REPO, 'contracts', 'fixtures');
|
||||
|
||||
const PLACEHOLDERS = new Set(['$uuid', '$isoDate', '$any', '$opaque', '$taskId', '$taskId2']);
|
||||
|
||||
/**
|
||||
* Concrete stand-ins for the placeholders, used when a golden payload is validated on its
|
||||
* own. A validator applies length and pattern rules, so "$opaque" has to become something
|
||||
* token-shaped before it is checked.
|
||||
*/
|
||||
const CONCRETE: Record<string, string> = {
|
||||
$uuid: 'e6f0a1b2-3c4d-4e5f-8a9b-0c1d2e3f4a5b',
|
||||
$taskId: 'e6f0a1b2-3c4d-4e5f-8a9b-0c1d2e3f4a5b',
|
||||
$taskId2: '11112222-3333-4444-8555-666677778888',
|
||||
$isoDate: '2026-09-09T10:14:52Z',
|
||||
$any: 'placeholder',
|
||||
$opaque: 'cGxhY2Vob2xkZXItdG9rZW4tNjQtYnl0ZXMtb2YtZW50cm9weS1nb2VzLWhlcmU',
|
||||
};
|
||||
|
||||
function concrete(value: unknown): unknown {
|
||||
if (typeof value === 'string') return CONCRETE[value] ?? value;
|
||||
if (Array.isArray(value)) return value.map(concrete);
|
||||
if (value && typeof value === 'object') {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [k, v] of Object.entries(value as Record<string, unknown>)) out[k] = concrete(v);
|
||||
return out;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
interface Fixture {
|
||||
file: string;
|
||||
name: string;
|
||||
kind?: 'timeout';
|
||||
/** A condition the server cannot produce from the request alone. Skipped unless the
|
||||
* harness has arranged it — see tests/integration. */
|
||||
requires?: string;
|
||||
transport?: TransportName;
|
||||
deadlineMs?: number;
|
||||
request?: { jsonrpc: '2.0'; id: number | string; method: string; params?: unknown };
|
||||
notification?: { jsonrpc: '2.0'; method: string; params: unknown };
|
||||
response?: { jsonrpc: '2.0'; id: number | string; result?: unknown; error?: { code: number } } | null;
|
||||
}
|
||||
|
||||
interface Outcome {
|
||||
fixture: string;
|
||||
transport: TransportName | 'static';
|
||||
ok: boolean;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- shape match
|
||||
|
||||
/**
|
||||
* Compare an actual value against a golden one structurally. Placeholders match anything;
|
||||
* objects must have the same keys; arrays must agree on emptiness and on element shape.
|
||||
*/
|
||||
function shapeMismatch(golden: unknown, actual: unknown, path = ''): string | null {
|
||||
if (typeof golden === 'string' && PLACEHOLDERS.has(golden)) return null;
|
||||
// The generated validator has already ruled on whether null is allowed here, so a null
|
||||
// is never a shape failure: a golden file shows one plausible value, not the only one.
|
||||
if (actual === null) return null;
|
||||
if (golden === null) return actual === null ? null : `${path}: expected null, got ${typeName(actual)}`;
|
||||
if (Array.isArray(golden)) {
|
||||
if (!Array.isArray(actual)) return `${path}: expected an array, got ${typeName(actual)}`;
|
||||
if (golden.length > 0 && actual.length === 0) return `${path}: expected a non-empty array`;
|
||||
if (golden.length > 0 && actual.length > 0) return shapeMismatch(golden[0], actual[0], `${path}/0`);
|
||||
return null;
|
||||
}
|
||||
if (typeof golden === 'object') {
|
||||
if (typeof actual !== 'object' || actual === null || Array.isArray(actual))
|
||||
return `${path}: expected an object, got ${typeName(actual)}`;
|
||||
const g = golden as Record<string, unknown>;
|
||||
const a = actual as Record<string, unknown>;
|
||||
for (const key of Object.keys(g)) {
|
||||
// A golden null means "may be absent"; the contract treats absent and null alike.
|
||||
if (!(key in a)) {
|
||||
if (g[key] === null) continue;
|
||||
return `${path}/${key}: missing from the response`;
|
||||
}
|
||||
const sub = shapeMismatch(g[key], a[key], `${path}/${key}`);
|
||||
if (sub) return sub;
|
||||
}
|
||||
for (const key of Object.keys(a)) {
|
||||
if (!(key in g)) return `${path}/${key}: not in the contract's result`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (typeof golden !== typeof actual) return `${path}: expected ${typeof golden}, got ${typeName(actual)}`;
|
||||
return null;
|
||||
}
|
||||
|
||||
function typeName(v: unknown): string {
|
||||
if (v === null) return 'null';
|
||||
if (Array.isArray(v)) return 'array';
|
||||
return typeof v;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------- fixtures
|
||||
|
||||
function walk(dir: string): string[] {
|
||||
const out: string[] = [];
|
||||
for (const entry of readdirSync(dir)) {
|
||||
const full = join(dir, entry);
|
||||
if (statSync(full).isDirectory()) out.push(...walk(full));
|
||||
else if (entry.endsWith('.json')) out.push(full);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function loadFixtures(): Fixture[] {
|
||||
return walk(FIXTURES).map((file) => ({
|
||||
...(JSON.parse(readFileSync(file, 'utf8')) as Omit<Fixture, 'file'>),
|
||||
file: relative(REPO, file),
|
||||
}));
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------- checks
|
||||
|
||||
/** Runs with no server: the generated validators must accept every golden payload. */
|
||||
function staticChecks(fixtures: readonly Fixture[]): Outcome[] {
|
||||
const out: Outcome[] = [];
|
||||
for (const f of fixtures) {
|
||||
if (f.notification) {
|
||||
const name = f.notification.method;
|
||||
if (!isEventName(name)) {
|
||||
out.push({ fixture: f.file, transport: 'static', ok: false, detail: `unknown event ${name}` });
|
||||
continue;
|
||||
}
|
||||
const r = validateEventParams(name, concrete(f.notification.params));
|
||||
out.push({ fixture: f.file, transport: 'static', ok: r.ok,
|
||||
detail: r.ok ? 'event payload validates' : `${r.path}: ${r.message}` });
|
||||
continue;
|
||||
}
|
||||
const method = f.request?.method;
|
||||
if (method === undefined || !isMethodName(method)) continue;
|
||||
|
||||
const expectsInvalidParams = f.response?.error?.code === -32602;
|
||||
const r = validateParams(method, concrete(f.request?.params ?? {}));
|
||||
if (expectsInvalidParams) {
|
||||
out.push({ fixture: f.file, transport: 'static', ok: !r.ok,
|
||||
detail: r.ok ? 'expects -32602 but the params validate' : 'params correctly rejected' });
|
||||
} else {
|
||||
out.push({ fixture: f.file, transport: 'static', ok: r.ok,
|
||||
detail: r.ok ? 'params validate' : `${r.path}: ${r.message}` });
|
||||
}
|
||||
|
||||
if (f.response && 'result' in f.response) {
|
||||
const rr = validateResult(method, concrete(f.response.result));
|
||||
out.push({ fixture: f.file, transport: 'static', ok: rr.ok,
|
||||
detail: rr.ok ? 'golden result validates' : `${rr.path}: ${rr.message}` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Methods that destroy the state later fixtures rely on. Replayed last so the suite does
|
||||
* not depend on file order, which is the sort of thing that goes green locally and red in
|
||||
* CI on a different filesystem.
|
||||
*/
|
||||
const DESTRUCTIVE = new Set<string>(['download.remove']);
|
||||
|
||||
function replayOrder(a: Fixture, b: Fixture): number {
|
||||
const rank = (f: Fixture): number => (DESTRUCTIVE.has(f.request?.method ?? '') ? 1 : 0);
|
||||
return rank(a) - rank(b) || a.file.localeCompare(b.file);
|
||||
}
|
||||
|
||||
/** Substitute the ids the runner bound during setup into a fixture's params. */
|
||||
function bind(value: unknown, bindings: Record<string, string>): unknown {
|
||||
if (typeof value === 'string') return bindings[value] ?? value;
|
||||
if (Array.isArray(value)) return value.map((v) => bind(v, bindings));
|
||||
if (value && typeof value === 'object') {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [k, v] of Object.entries(value as Record<string, unknown>)) out[k] = bind(v, bindings);
|
||||
return out;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the tasks the task-referencing fixtures bind to. Doing this per connection is
|
||||
* what lets the same suite run against an empty veloxd and against a seeded mockd.
|
||||
*/
|
||||
async function setupBindings(conn: Conn): Promise<Record<string, string>> {
|
||||
const bindings: Record<string, string> = {};
|
||||
for (const [key, url] of [['$taskId', 'https://example.org/conformance-a.bin'],
|
||||
['$taskId2', 'https://example.org/conformance-b.bin']] as const) {
|
||||
const added = await conn.call('download.add', { url, startMode: 'later' });
|
||||
bindings[key] = added.taskId;
|
||||
}
|
||||
return bindings;
|
||||
}
|
||||
|
||||
async function replay(conn: Conn, fixtures: readonly Fixture[],
|
||||
bindings: Record<string, string>,
|
||||
includeRequires = false): Promise<Outcome[]> {
|
||||
const out: Outcome[] = [];
|
||||
const t = conn.transport;
|
||||
|
||||
for (const f of [...fixtures].sort(replayOrder)) {
|
||||
if (!f.request) continue;
|
||||
const method = f.request.method;
|
||||
if (f.transport !== undefined && f.transport !== t) continue;
|
||||
if (!isMethodName(method)) continue;
|
||||
if (!(METHODS[method].transports as readonly string[]).includes(t)) continue;
|
||||
if (f.requires !== undefined && !includeRequires) {
|
||||
out.push({ fixture: f.file, transport: t, ok: true,
|
||||
detail: `skipped: requires ${f.requires}` });
|
||||
continue;
|
||||
}
|
||||
|
||||
const deadline = f.deadlineMs ?? Math.max(METHODS[method].deadlineMs, 2000);
|
||||
const frame = await conn.request(method, bind(f.request.params ?? {}, bindings), deadline);
|
||||
|
||||
if (f.kind === 'timeout') {
|
||||
out.push({
|
||||
fixture: f.file, transport: t, ok: frame === null,
|
||||
detail: frame === null
|
||||
? `no reply within ${deadline} ms — the client fails open, as it must`
|
||||
: 'the server answered a fixture that requires silence',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (frame === null) {
|
||||
out.push({ fixture: f.file, transport: t, ok: false, detail: `no reply within ${deadline} ms` });
|
||||
continue;
|
||||
}
|
||||
|
||||
const expected = f.response;
|
||||
if (expected && 'error' in expected && expected.error) {
|
||||
const got = frame.error?.code;
|
||||
out.push({
|
||||
fixture: f.file, transport: t, ok: got === expected.error.code,
|
||||
detail: got === expected.error.code
|
||||
? `error ${got} as documented`
|
||||
: `expected error ${expected.error.code}, got ${frame.error ? `error ${got}` : 'a result'}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (frame.error) {
|
||||
out.push({ fixture: f.file, transport: t, ok: false,
|
||||
detail: `expected a result, got error ${frame.error.code}: ${frame.error.message}` });
|
||||
continue;
|
||||
}
|
||||
|
||||
const validated = validateResult(method, frame.result);
|
||||
if (!validated.ok) {
|
||||
out.push({ fixture: f.file, transport: t, ok: false,
|
||||
detail: `result fails the generated validator at ${validated.path}: ${validated.message}` });
|
||||
continue;
|
||||
}
|
||||
const mismatch = expected && 'result' in expected
|
||||
? shapeMismatch(expected.result, frame.result)
|
||||
: null;
|
||||
out.push({ fixture: f.file, transport: t, ok: mismatch === null,
|
||||
detail: mismatch ?? 'result validates and matches the golden shape' });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The transport rules are part of the contract, so they get replayed too. */
|
||||
async function privilegeChecks(conn: Conn): Promise<Outcome[]> {
|
||||
if (conn.transport !== 'ws') return [];
|
||||
const out: Outcome[] = [];
|
||||
const privileged = (Object.keys(METHODS) as MethodName[]).filter((m) => METHODS[m].privileged);
|
||||
for (const method of privileged) {
|
||||
const frame = await conn.request(method, {}, 3000);
|
||||
const ok = frame?.error?.code === -32003;
|
||||
out.push({
|
||||
fixture: `transport-rules/${method}`, transport: 'ws', ok,
|
||||
detail: ok ? 'refused with -32003 over the WebSocket, as required'
|
||||
: `expected -32003, got ${frame ? JSON.stringify(frame.error ?? frame.result).slice(0, 80) : 'no reply'}`,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------- main
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const argv = process.argv.slice(2);
|
||||
const arg = (name: string): string | undefined => {
|
||||
const i = argv.indexOf(name);
|
||||
return i === -1 ? undefined : argv[i + 1];
|
||||
};
|
||||
|
||||
// --only narrows the run to fixtures whose path contains a substring, and
|
||||
// --include-requires replays the ones needing a condition the harness has arranged
|
||||
// (a slow daemon, a hostile origin server). run.sh uses both to prove capture.offer
|
||||
// fails open, which cannot be shown against a healthy server.
|
||||
const only = arg('--only');
|
||||
const includeRequires = argv.includes('--include-requires');
|
||||
const all = loadFixtures();
|
||||
const fixtures = only === undefined ? all : all.filter((f) => f.file.includes(only));
|
||||
if (fixtures.length === 0) {
|
||||
process.stderr.write(`conformance: --only ${String(only)} matched no fixtures\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
const results: Outcome[] = [...staticChecks(fixtures)];
|
||||
|
||||
const udsPath = arg('--uds');
|
||||
const wsPort = arg('--ws-port');
|
||||
|
||||
if (udsPath) {
|
||||
const conn = await connectUds(udsPath);
|
||||
await conn.call('session.hello', { clientType: 'test', clientName: 'conformance', protocolVersion: '1.0.0' });
|
||||
results.push(...(await replay(conn, fixtures, await setupBindings(conn), includeRequires)));
|
||||
conn.close();
|
||||
}
|
||||
if (wsPort) {
|
||||
const conn = await connectWs(Number(wsPort));
|
||||
const paired = await conn.request(
|
||||
'session.pair',
|
||||
{ clientName: 'conformance', extensionId: '11111111-2222-3333-4444-555555555555' },
|
||||
5000,
|
||||
);
|
||||
const token = (paired?.result as { token?: string } | undefined)?.token;
|
||||
if (token === undefined) throw new Error('pairing failed: no token issued');
|
||||
await conn.request('session.hello',
|
||||
{ clientType: 'test', clientName: 'conformance', protocolVersion: '1.0.0', token }, 5000);
|
||||
results.push(...(await replay(conn, fixtures, await setupBindings(conn), includeRequires)));
|
||||
results.push(...(await privilegeChecks(conn)));
|
||||
conn.close();
|
||||
}
|
||||
if (!udsPath && !wsPort) {
|
||||
process.stdout.write('no --uds or --ws-port given: ran static checks only\n');
|
||||
}
|
||||
|
||||
const failed = results.filter((r) => !r.ok);
|
||||
for (const r of failed) {
|
||||
process.stdout.write(`FAIL [${r.transport}] ${r.fixture}\n ${r.detail}\n`);
|
||||
}
|
||||
const byTransport = new Map<string, number>();
|
||||
for (const r of results) byTransport.set(r.transport, (byTransport.get(r.transport) ?? 0) + 1);
|
||||
const summary = [...byTransport].map(([k, v]) => `${k}:${v}`).join(' ');
|
||||
process.stdout.write(`\n${results.length - failed.length}/${results.length} checks passed (${summary})\n`);
|
||||
process.exit(failed.length === 0 ? 0 : 1);
|
||||
}
|
||||
|
||||
main().catch((err: unknown) => {
|
||||
process.stderr.write(`conformance: ${String(err)}\n`);
|
||||
process.exit(2);
|
||||
});
|
||||
Reference in New Issue
Block a user