vdm/tests/conformance/ts/replay.ts

/**
 * Conformance runner, TypeScript side.
 *
 * Replays every fixture in contracts/fixtures against a live server — mockd today, veloxd
 * from M1 — through the generated client types and validators. The same suite runs against
 * both, which is the point: if a lane drifts from the contract, this goes red the same day
 * rather than at M2 integration.
 *
 * What each fixture asserts
 *   success   the reply carries a result; the result passes the generated validator; its
 *             shape matches the golden file
 *   error     the reply carries an error with the fixture's code
 *   timeout   nothing arrives inside the deadline, and the client is expected to give up.
 *             This is capture.offer's fail-open guarantee, and it is a pass when the
 *             server stays silent.
 *
 * Values are compared by *shape*, not by equality: a live daemon returns its own task ids
 * and its own clock, and demanding byte-identical results would only teach the suite to
 * lie. Types, key sets and error codes are compared exactly.
 *
 *   npx tsx replay.ts --uds /run/user/1000/velox/velox.sock --ws-port 52000
 */

import { readFileSync, readdirSync, statSync } from 'node:fs';
import { join, relative, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';

import { connectUds, connectWs, type Conn, type TransportName } from './client.js';
import {
  METHODS,
  isMethodName,
  type MethodName,
} from '../../../extension/src/shared/protocol/methods.js';
import { isEventName } from '../../../extension/src/shared/protocol/events.js';
import {
  validateEventParams,
  validateParams,
  validateResult,
} from '../../../extension/src/shared/protocol/validate.js';

const HERE = resolve(fileURLToPath(import.meta.url), '..');
const REPO = resolve(HERE, '..', '..', '..');
const FIXTURES = resolve(REPO, 'contracts', 'fixtures');

const PLACEHOLDERS = new Set(['$uuid', '$isoDate', '$any', '$opaque', '$taskId', '$taskId2']);

/**
 * Concrete stand-ins for the placeholders, used when a golden payload is validated on its
 * own. A validator applies length and pattern rules, so "$opaque" has to become something
 * token-shaped before it is checked.
 */
const CONCRETE: Record<string, string> = {
  $uuid: 'e6f0a1b2-3c4d-4e5f-8a9b-0c1d2e3f4a5b',
  $taskId: 'e6f0a1b2-3c4d-4e5f-8a9b-0c1d2e3f4a5b',
  $taskId2: '11112222-3333-4444-8555-666677778888',
  $isoDate: '2026-09-09T10:14:52Z',
  $any: 'placeholder',
  $opaque: 'cGxhY2Vob2xkZXItdG9rZW4tNjQtYnl0ZXMtb2YtZW50cm9weS1nb2VzLWhlcmU',
};

function concrete(value: unknown): unknown {
  if (typeof value === 'string') return CONCRETE[value] ?? value;
  if (Array.isArray(value)) return value.map(concrete);
  if (value && typeof value === 'object') {
    const out: Record<string, unknown> = {};
    for (const [k, v] of Object.entries(value as Record<string, unknown>)) out[k] = concrete(v);
    return out;
  }
  return value;
}

interface Fixture {
  file: string;
  name: string;
  kind?: 'timeout';
  /** A condition the server cannot produce from the request alone. Skipped unless the
   *  harness has arranged it — see tests/integration. */
  requires?: string;
  transport?: TransportName;
  deadlineMs?: number;
  request?: { jsonrpc: '2.0'; id: number | string; method: string; params?: unknown };
  notification?: { jsonrpc: '2.0'; method: string; params: unknown };
  response?: { jsonrpc: '2.0'; id: number | string; result?: unknown; error?: { code: number } } | null;
}

interface Outcome {
  fixture: string;
  transport: TransportName | 'static';
  ok: boolean;
  detail: string;
}

// ---------------------------------------------------------------- shape match

/**
 * Compare an actual value against a golden one structurally. Placeholders match anything;
 * objects must have the same keys; arrays must agree on emptiness and on element shape.
 */
function shapeMismatch(golden: unknown, actual: unknown, path = ''): string | null {
  if (typeof golden === 'string' && PLACEHOLDERS.has(golden)) return null;
  // The generated validator has already ruled on whether null is allowed here, so a null
  // is never a shape failure: a golden file shows one plausible value, not the only one.
  if (actual === null) return null;
  if (golden === null) return actual === null ? null : `${path}: expected null, got ${typeName(actual)}`;
  if (Array.isArray(golden)) {
    if (!Array.isArray(actual)) return `${path}: expected an array, got ${typeName(actual)}`;
    if (golden.length > 0 && actual.length === 0) return `${path}: expected a non-empty array`;
    if (golden.length > 0 && actual.length > 0) return shapeMismatch(golden[0], actual[0], `${path}/0`);
    return null;
  }
  if (typeof golden === 'object') {
    if (typeof actual !== 'object' || actual === null || Array.isArray(actual))
      return `${path}: expected an object, got ${typeName(actual)}`;
    const g = golden as Record<string, unknown>;
    const a = actual as Record<string, unknown>;
    for (const key of Object.keys(g)) {
      // A golden null means "may be absent"; the contract treats absent and null alike.
      if (!(key in a)) {
        if (g[key] === null) continue;
        return `${path}/${key}: missing from the response`;
      }
      const sub = shapeMismatch(g[key], a[key], `${path}/${key}`);
      if (sub) return sub;
    }
    for (const key of Object.keys(a)) {
      if (!(key in g)) return `${path}/${key}: not in the contract's result`;
    }
    return null;
  }
  if (typeof golden !== typeof actual) return `${path}: expected ${typeof golden}, got ${typeName(actual)}`;
  return null;
}

function typeName(v: unknown): string {
  if (v === null) return 'null';
  if (Array.isArray(v)) return 'array';
  return typeof v;
}

// ------------------------------------------------------------------- fixtures

function walk(dir: string): string[] {
  const out: string[] = [];
  for (const entry of readdirSync(dir)) {
    const full = join(dir, entry);
    if (statSync(full).isDirectory()) out.push(...walk(full));
    else if (entry.endsWith('.json')) out.push(full);
  }
  return out;
}

function loadFixtures(): Fixture[] {
  return walk(FIXTURES).map((file) => ({
    ...(JSON.parse(readFileSync(file, 'utf8')) as Omit<Fixture, 'file'>),
    file: relative(REPO, file),
  }));
}

// -------------------------------------------------------------------- checks

/** Runs with no server: the generated validators must accept every golden payload. */
function staticChecks(fixtures: readonly Fixture[]): Outcome[] {
  const out: Outcome[] = [];
  for (const f of fixtures) {
    if (f.notification) {
      const name = f.notification.method;
      if (!isEventName(name)) {
        out.push({ fixture: f.file, transport: 'static', ok: false, detail: `unknown event ${name}` });
        continue;
      }
      const r = validateEventParams(name, concrete(f.notification.params));
      out.push({ fixture: f.file, transport: 'static', ok: r.ok,
                 detail: r.ok ? 'event payload validates' : `${r.path}: ${r.message}` });
      continue;
    }
    const method = f.request?.method;
    if (method === undefined || !isMethodName(method)) continue;

    const expectsInvalidParams = f.response?.error?.code === -32602;
    const r = validateParams(method, concrete(f.request?.params ?? {}));
    if (expectsInvalidParams) {
      out.push({ fixture: f.file, transport: 'static', ok: !r.ok,
                 detail: r.ok ? 'expects -32602 but the params validate' : 'params correctly rejected' });
    } else {
      out.push({ fixture: f.file, transport: 'static', ok: r.ok,
                 detail: r.ok ? 'params validate' : `${r.path}: ${r.message}` });
    }

    if (f.response && 'result' in f.response) {
      const rr = validateResult(method, concrete(f.response.result));
      out.push({ fixture: f.file, transport: 'static', ok: rr.ok,
                 detail: rr.ok ? 'golden result validates' : `${rr.path}: ${rr.message}` });
    }
  }
  return out;
}

/**
 * Methods that destroy the state later fixtures rely on. Replayed last so the suite does
 * not depend on file order, which is the sort of thing that goes green locally and red in
 * CI on a different filesystem.
 */
const DESTRUCTIVE = new Set<string>(['download.remove']);

function replayOrder(a: Fixture, b: Fixture): number {
  const rank = (f: Fixture): number => (DESTRUCTIVE.has(f.request?.method ?? '') ? 1 : 0);
  return rank(a) - rank(b) || a.file.localeCompare(b.file);
}

/** Substitute the ids the runner bound during setup into a fixture's params. */
function bind(value: unknown, bindings: Record<string, string>): unknown {
  if (typeof value === 'string') return bindings[value] ?? value;
  if (Array.isArray(value)) return value.map((v) => bind(v, bindings));
  if (value && typeof value === 'object') {
    const out: Record<string, unknown> = {};
    for (const [k, v] of Object.entries(value as Record<string, unknown>)) out[k] = bind(v, bindings);
    return out;
  }
  return value;
}

/**
 * Create the tasks the task-referencing fixtures bind to. Doing this per connection is
 * what lets the same suite run against an empty veloxd and against a seeded mockd.
 */
async function setupBindings(conn: Conn): Promise<Record<string, string>> {
  const bindings: Record<string, string> = {};
  for (const [key, url] of [['$taskId', 'https://example.org/conformance-a.bin'],
                            ['$taskId2', 'https://example.org/conformance-b.bin']] as const) {
    const added = await conn.call('download.add', { url, startMode: 'later' });
    bindings[key] = added.taskId;
  }
  return bindings;
}

async function replay(conn: Conn, fixtures: readonly Fixture[],
                      bindings: Record<string, string>,
                      includeRequires = false): Promise<Outcome[]> {
  const out: Outcome[] = [];
  const t = conn.transport;

  for (const f of [...fixtures].sort(replayOrder)) {
    if (!f.request) continue;
    const method = f.request.method;
    if (f.transport !== undefined && f.transport !== t) continue;
    if (!isMethodName(method)) continue;
    if (!(METHODS[method].transports as readonly string[]).includes(t)) continue;
    if (f.requires !== undefined && !includeRequires) {
      out.push({ fixture: f.file, transport: t, ok: true,
                 detail: `skipped: requires ${f.requires}` });
      continue;
    }

    const deadline = f.deadlineMs ?? Math.max(METHODS[method].deadlineMs, 2000);
    const frame = await conn.request(method, bind(f.request.params ?? {}, bindings), deadline);

    if (f.kind === 'timeout') {
      out.push({
        fixture: f.file, transport: t, ok: frame === null,
        detail: frame === null
          ? `no reply within ${deadline} ms — the client fails open, as it must`
          : 'the server answered a fixture that requires silence',
      });
      continue;
    }

    if (frame === null) {
      out.push({ fixture: f.file, transport: t, ok: false, detail: `no reply within ${deadline} ms` });
      continue;
    }

    const expected = f.response;
    if (expected && 'error' in expected && expected.error) {
      const got = frame.error?.code;
      out.push({
        fixture: f.file, transport: t, ok: got === expected.error.code,
        detail: got === expected.error.code
          ? `error ${got} as documented`
          : `expected error ${expected.error.code}, got ${frame.error ? `error ${got}` : 'a result'}`,
      });
      continue;
    }

    if (frame.error) {
      out.push({ fixture: f.file, transport: t, ok: false,
                 detail: `expected a result, got error ${frame.error.code}: ${frame.error.message}` });
      continue;
    }

    const validated = validateResult(method, frame.result);
    if (!validated.ok) {
      out.push({ fixture: f.file, transport: t, ok: false,
                 detail: `result fails the generated validator at ${validated.path}: ${validated.message}` });
      continue;
    }
    const mismatch = expected && 'result' in expected
      ? shapeMismatch(expected.result, frame.result)
      : null;
    out.push({ fixture: f.file, transport: t, ok: mismatch === null,
               detail: mismatch ?? 'result validates and matches the golden shape' });
  }
  return out;
}

/** The transport rules are part of the contract, so they get replayed too. */
async function privilegeChecks(conn: Conn): Promise<Outcome[]> {
  if (conn.transport !== 'ws') return [];
  const out: Outcome[] = [];
  const privileged = (Object.keys(METHODS) as MethodName[]).filter((m) => METHODS[m].privileged);
  for (const method of privileged) {
    const frame = await conn.request(method, {}, 3000);
    const ok = frame?.error?.code === -32003;
    out.push({
      fixture: `transport-rules/${method}`, transport: 'ws', ok,
      detail: ok ? 'refused with -32003 over the WebSocket, as required'
                 : `expected -32003, got ${frame ? JSON.stringify(frame.error ?? frame.result).slice(0, 80) : 'no reply'}`,
    });
  }
  return out;
}

// ---------------------------------------------------------------------- main

async function main(): Promise<void> {
  const argv = process.argv.slice(2);
  const arg = (name: string): string | undefined => {
    const i = argv.indexOf(name);
    return i === -1 ? undefined : argv[i + 1];
  };

  // --only narrows the run to fixtures whose path contains a substring, and
  // --include-requires replays the ones needing a condition the harness has arranged
  // (a slow daemon, a hostile origin server). run.sh uses both to prove capture.offer
  // fails open, which cannot be shown against a healthy server.
  const only = arg('--only');
  const includeRequires = argv.includes('--include-requires');
  const all = loadFixtures();
  const fixtures = only === undefined ? all : all.filter((f) => f.file.includes(only));
  if (fixtures.length === 0) {
    process.stderr.write(`conformance: --only ${String(only)} matched no fixtures\n`);
    process.exit(2);
  }
  const results: Outcome[] = [...staticChecks(fixtures)];

  const udsPath = arg('--uds');
  const wsPort = arg('--ws-port');

  if (udsPath) {
    const conn = await connectUds(udsPath);
    await conn.call('session.hello', { clientType: 'test', clientName: 'conformance', protocolVersion: '1.0.0' });
    results.push(...(await replay(conn, fixtures, await setupBindings(conn), includeRequires)));
    conn.close();
  }
  if (wsPort) {
    const conn = await connectWs(Number(wsPort));
    const paired = await conn.request(
      'session.pair',
      { clientName: 'conformance', extensionId: '11111111-2222-3333-4444-555555555555' },
      5000,
    );
    const token = (paired?.result as { token?: string } | undefined)?.token;
    if (token === undefined) throw new Error('pairing failed: no token issued');
    await conn.request('session.hello',
      { clientType: 'test', clientName: 'conformance', protocolVersion: '1.0.0', token }, 5000);
    results.push(...(await replay(conn, fixtures, await setupBindings(conn), includeRequires)));
    results.push(...(await privilegeChecks(conn)));
    conn.close();
  }
  if (!udsPath && !wsPort) {
    process.stdout.write('no --uds or --ws-port given: ran static checks only\n');
  }

  const failed = results.filter((r) => !r.ok);
  for (const r of failed) {
    process.stdout.write(`FAIL [${r.transport}] ${r.fixture}\n       ${r.detail}\n`);
  }
  const byTransport = new Map<string, number>();
  for (const r of results) byTransport.set(r.transport, (byTransport.get(r.transport) ?? 0) + 1);
  const summary = [...byTransport].map(([k, v]) => `${k}:${v}`).join('  ');
  process.stdout.write(`\n${results.length - failed.length}/${results.length} checks passed  (${summary})\n`);
  process.exit(failed.length === 0 ? 0 : 1);
}

main().catch((err: unknown) => {
  process.stderr.write(`conformance: ${String(err)}\n`);
  process.exit(2);
});