vdm/tests/conformance/ts/replay.ts

/**
 * Conformance runner, TypeScript side.
 *
 * Replays every fixture in contracts/fixtures against a live server — mockd today, veloxd
 * from M1 — through the generated client types and validators. The same suite runs against
 * both, which is the point: if a lane drifts from the contract, this goes red the same day
 * rather than at M2 integration.
 *
 * What each fixture asserts
 *   success   the reply carries a result; the result passes the generated validator; its
 *             shape matches the golden file
 *   error     the reply carries an error with the fixture's code
 *   timeout   nothing arrives inside the deadline, and the client is expected to give up.
 *             This is capture.offer's fail-open guarantee, and it is a pass when the
 *             server stays silent.
 *
 * Values are compared by *shape*, not by equality: a live daemon returns its own task ids
 * and its own clock, and demanding byte-identical results would only teach the suite to
 * lie. Types, key sets and error codes are compared exactly.
 *
 *   npx tsx replay.ts --uds /run/user/1000/velox/velox.sock --ws-port 52000
 */

import { readFileSync, readdirSync, statSync } from 'node:fs';
import { join, relative, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';

import { connectUds, connectWs, type Conn, type TransportName } from './client.js';
import {
  METHODS,
  isMethodName,
  type MethodName,
} from '../../../extension/src/shared/protocol/methods.js';
import { isEventName } from '../../../extension/src/shared/protocol/events.js';
import {
  validateEventParams,
  validateParams,
  validateResult,
} from '../../../extension/src/shared/protocol/validate.js';

const HERE = resolve(fileURLToPath(import.meta.url), '..');
const REPO = resolve(HERE, '..', '..', '..');
const FIXTURES = resolve(REPO, 'contracts', 'fixtures');

const PLACEHOLDERS = new Set(['$uuid', '$isoDate', '$any', '$opaque', '$taskId', '$taskId2']);

/**
 * Concrete stand-ins for the placeholders, used when a golden payload is validated on its
 * own. A validator applies length and pattern rules, so "$opaque" has to become something
 * token-shaped before it is checked.
 */
const CONCRETE: Record<string, string> = {
  $uuid: 'e6f0a1b2-3c4d-4e5f-8a9b-0c1d2e3f4a5b',
  $taskId: 'e6f0a1b2-3c4d-4e5f-8a9b-0c1d2e3f4a5b',
  $taskId2: '11112222-3333-4444-8555-666677778888',
  $isoDate: '2026-09-09T10:14:52Z',
  $any: 'placeholder',
  $opaque: 'cGxhY2Vob2xkZXItdG9rZW4tNjQtYnl0ZXMtb2YtZW50cm9weS1nb2VzLWhlcmU',
};

function concrete(value: unknown): unknown {
  if (typeof value === 'string') return CONCRETE[value] ?? value;
  if (Array.isArray(value)) return value.map(concrete);
  if (value && typeof value === 'object') {
    const out: Record<string, unknown> = {};
    for (const [k, v] of Object.entries(value as Record<string, unknown>)) out[k] = concrete(v);
    return out;
  }
  return value;
}

interface Fixture {
  file: string;
  name: string;
  kind?: 'timeout';
  /** A condition the server cannot produce from the request alone. Skipped unless the
   *  harness has arranged it — see tests/integration. */
  requires?: string;
  /** This request is documented to make the *server* close the connection after replying
   *  (e.g. a mismatched protocol major on the Unix socket). replay() reconnects afterward
   *  so every later fixture in the shared-connection replay isn't sent into a dead socket
   *  and left to time out one by one — which is silent when the fixture in question is
   *  also on the xfail allowlist, since applyXfail accepts any failure reason. */
  closesConnection?: boolean;
  transport?: TransportName;
  deadlineMs?: number;
  request?: { jsonrpc: '2.0'; id: number | string; method: string; params?: unknown };
  notification?: { jsonrpc: '2.0'; method: string; params: unknown };
  response?: { jsonrpc: '2.0'; id: number | string; result?: unknown; error?: { code: number } } | null;
}

interface Outcome {
  fixture: string;
  transport: TransportName | 'static';
  ok: boolean;
  detail: string;
}

// ---------------------------------------------------------------- shape match

/**
 * Compare an actual value against a golden one structurally. Placeholders match anything;
 * objects must have the same keys; arrays must agree on emptiness and on element shape.
 */
function shapeMismatch(golden: unknown, actual: unknown, path = ''): string | null {
  if (typeof golden === 'string' && PLACEHOLDERS.has(golden)) return null;
  // The generated validator has already ruled on whether null is allowed here, so a null
  // is never a shape failure: a golden file shows one plausible value, not the only one.
  if (actual === null) return null;
  if (golden === null) return actual === null ? null : `${path}: expected null, got ${typeName(actual)}`;
  if (Array.isArray(golden)) {
    if (!Array.isArray(actual)) return `${path}: expected an array, got ${typeName(actual)}`;
    if (golden.length > 0 && actual.length === 0) return `${path}: expected a non-empty array`;
    if (golden.length > 0 && actual.length > 0) return shapeMismatch(golden[0], actual[0], `${path}/0`);
    return null;
  }
  if (typeof golden === 'object') {
    if (typeof actual !== 'object' || actual === null || Array.isArray(actual))
      return `${path}: expected an object, got ${typeName(actual)}`;
    const g = golden as Record<string, unknown>;
    const a = actual as Record<string, unknown>;
    for (const key of Object.keys(g)) {
      // A golden null means "may be absent"; the contract treats absent and null alike.
      if (!(key in a)) {
        if (g[key] === null) continue;
        return `${path}/${key}: missing from the response`;
      }
      const sub = shapeMismatch(g[key], a[key], `${path}/${key}`);
      if (sub) return sub;
    }
    for (const key of Object.keys(a)) {
      if (!(key in g)) return `${path}/${key}: not in the contract's result`;
    }
    return null;
  }
  if (typeof golden !== typeof actual) return `${path}: expected ${typeof golden}, got ${typeName(actual)}`;
  return null;
}

function typeName(v: unknown): string {
  if (v === null) return 'null';
  if (Array.isArray(v)) return 'array';
  return typeof v;
}

// ------------------------------------------------------------------- fixtures

function walk(dir: string): string[] {
  const out: string[] = [];
  for (const entry of readdirSync(dir)) {
    const full = join(dir, entry);
    if (statSync(full).isDirectory()) out.push(...walk(full));
    else if (entry.endsWith('.json')) out.push(full);
  }
  return out;
}

function loadFixtures(): Fixture[] {
  return walk(FIXTURES).map((file) => ({
    ...(JSON.parse(readFileSync(file, 'utf8')) as Omit<Fixture, 'file'>),
    file: relative(REPO, file),
  }));
}

// -------------------------------------------------------------------- checks

/** Runs with no server: the generated validators must accept every golden payload. */
function staticChecks(fixtures: readonly Fixture[]): Outcome[] {
  const out: Outcome[] = [];
  for (const f of fixtures) {
    if (f.notification) {
      const name = f.notification.method;
      if (!isEventName(name)) {
        out.push({ fixture: f.file, transport: 'static', ok: false, detail: `unknown event ${name}` });
        continue;
      }
      const r = validateEventParams(name, concrete(f.notification.params));
      out.push({ fixture: f.file, transport: 'static', ok: r.ok,
                 detail: r.ok ? 'event payload validates' : `${r.path}: ${r.message}` });
      continue;
    }
    const method = f.request?.method;
    if (method === undefined || !isMethodName(method)) continue;

    const expectsInvalidParams = f.response?.error?.code === -32602;
    const r = validateParams(method, concrete(f.request?.params ?? {}));
    if (expectsInvalidParams) {
      out.push({ fixture: f.file, transport: 'static', ok: !r.ok,
                 detail: r.ok ? 'expects -32602 but the params validate' : 'params correctly rejected' });
    } else {
      out.push({ fixture: f.file, transport: 'static', ok: r.ok,
                 detail: r.ok ? 'params validate' : `${r.path}: ${r.message}` });
    }

    if (f.response && 'result' in f.response) {
      const rr = validateResult(method, concrete(f.response.result));
      out.push({ fixture: f.file, transport: 'static', ok: rr.ok,
                 detail: rr.ok ? 'golden result validates' : `${rr.path}: ${rr.message}` });
    }
  }
  return out;
}

/**
 * Methods that destroy state, or consume state another fixture creates. Replayed last so
 * the suite does not depend on file order, which is the sort of thing that goes green
 * locally and red in CI on a different filesystem — alphabetical happens to put
 * category.remove.json before category.upsert.json, and category.remove's fixture only
 * has a "firmware" category to delete because category.upsert's fixture just created one.
 */
const DESTRUCTIVE = new Set<string>(['download.remove', 'category.remove']);

function replayOrder(a: Fixture, b: Fixture): number {
  const rank = (f: Fixture): number => (DESTRUCTIVE.has(f.request?.method ?? '') ? 1 : 0);
  return rank(a) - rank(b) || a.file.localeCompare(b.file);
}

/** Substitute the ids the runner bound during setup into a fixture's params. */
function bind(value: unknown, bindings: Record<string, string>): unknown {
  if (typeof value === 'string') return bindings[value] ?? value;
  if (Array.isArray(value)) return value.map((v) => bind(v, bindings));
  if (value && typeof value === 'object') {
    const out: Record<string, unknown> = {};
    for (const [k, v] of Object.entries(value as Record<string, unknown>)) out[k] = bind(v, bindings);
    return out;
  }
  return value;
}

/**
 * Create the tasks the task-referencing fixtures bind to. Doing this per connection is
 * what lets the same suite run against an empty veloxd and against a seeded mockd.
 *
 * A server that cannot even complete this setup (not a fixture, so nothing above can
 * report it) still needs to be visible as a failure rather than an uncaught exception
 * that takes the whole runner down before a single fixture is checked — so a failure
 * here becomes an Outcome and setup moves on, leaving that binding unresolved (its
 * fixtures will then fail on the placeholder, individually, same as any other bad value).
 */
async function setupBindings(conn: Conn): Promise<{ bindings: Record<string, string>; setup: Outcome[] }> {
  const bindings: Record<string, string> = {};
  const setup: Outcome[] = [];
  for (const [key, url] of [['$taskId', 'https://example.org/conformance-a.bin'],
                            ['$taskId2', 'https://example.org/conformance-b.bin']] as const) {
    try {
      const added = await conn.call('download.add', { url, startMode: 'later' });
      bindings[key] = added.taskId;
      setup.push({ fixture: `setup/${key}`, transport: conn.transport, ok: true,
                   detail: 'download.add for the binding succeeded' });
    } catch (err) {
      setup.push({ fixture: `setup/${key}`, transport: conn.transport, ok: false,
                   detail: `download.add for the binding failed: ${String(err)}` });
    }
  }
  return { bindings, setup };
}

async function replay(initialConn: Conn, fixtures: readonly Fixture[],
                      bindings: Record<string, string>,
                      includeRequires = false,
                      reconnect?: () => Promise<Conn>):
    Promise<{ outcomes: Outcome[]; conn: Conn }> {
  const out: Outcome[] = [];
  let conn = initialConn;
  const t = conn.transport;

  for (const f of [...fixtures].sort(replayOrder)) {
    if (!f.request) continue;
    const method = f.request.method;
    if (f.transport !== undefined && f.transport !== t) continue;
    if (!isMethodName(method)) continue;
    if (!(METHODS[method].transports as readonly string[]).includes(t)) continue;
    if (f.requires !== undefined && !includeRequires) {
      out.push({ fixture: f.file, transport: t, ok: true,
                 detail: `skipped: requires ${f.requires}` });
      continue;
    }

    const deadline = f.deadlineMs ?? Math.max(METHODS[method].deadlineMs, 2000);
    if (process.env.DEBUG_CONFORMANCE) process.stderr.write(`>>> [${t}] ${f.file} ${method}\n`);
    const frame = await conn.request(method, bind(f.request.params ?? {}, bindings), deadline);
    if (process.env.DEBUG_CONFORMANCE) process.stderr.write(`<<< [${t}] ${f.file} ${frame ? 'ok' : 'TIMEOUT'}\n`);

    if (f.closesConnection && reconnect) {
      conn.close();
      conn = await reconnect();
    }

    if (f.kind === 'timeout') {
      out.push({
        fixture: f.file, transport: t, ok: frame === null,
        detail: frame === null
          ? `no reply within ${deadline} ms — the client fails open, as it must`
          : 'the server answered a fixture that requires silence',
      });
      continue;
    }

    if (frame === null) {
      out.push({ fixture: f.file, transport: t, ok: false, detail: `no reply within ${deadline} ms` });
      continue;
    }

    const expected = f.response;
    if (expected && 'error' in expected && expected.error) {
      const got = frame.error?.code;
      out.push({
        fixture: f.file, transport: t, ok: got === expected.error.code,
        detail: got === expected.error.code
          ? `error ${got} as documented`
          : `expected error ${expected.error.code}, got ${frame.error ? `error ${got}` : 'a result'}`,
      });
      continue;
    }

    if (frame.error) {
      out.push({ fixture: f.file, transport: t, ok: false,
                 detail: `expected a result, got error ${frame.error.code}: ${frame.error.message}` });
      continue;
    }

    const validated = validateResult(method, frame.result);
    if (!validated.ok) {
      out.push({ fixture: f.file, transport: t, ok: false,
                 detail: `result fails the generated validator at ${validated.path}: ${validated.message}` });
      continue;
    }
    const mismatch = expected && 'result' in expected
      ? shapeMismatch(expected.result, frame.result)
      : null;
    out.push({ fixture: f.file, transport: t, ok: mismatch === null,
               detail: mismatch ?? 'result validates and matches the golden shape' });
  }
  return { outcomes: out, conn };
}

/** The transport rules are part of the contract, so they get replayed too. */
async function privilegeChecks(conn: Conn): Promise<Outcome[]> {
  if (conn.transport !== 'ws') return [];
  const out: Outcome[] = [];
  const privileged = (Object.keys(METHODS) as MethodName[]).filter((m) => METHODS[m].privileged);
  for (const method of privileged) {
    const frame = await conn.request(method, {}, 3000);
    const ok = frame?.error?.code === -32003;
    out.push({
      fixture: `transport-rules/${method}`, transport: 'ws', ok,
      detail: ok ? 'refused with -32003 over the WebSocket, as required'
                 : `expected -32003, got ${frame ? JSON.stringify(frame.error ?? frame.result).slice(0, 80) : 'no reply'}`,
    });
  }
  return out;
}

// ----------------------------------------------------------- expected failure

/**
 * A fixture this runner is allowed to fail against the target server, with why. Used
 * against veloxd, which still has stub handlers (daemon/docs/deferrals.md D1-D4b) that
 * mockd does not: mockd always answers every fixture correctly, so this list is empty
 * there and the mechanism does not apply.
 *
 * `fixture` matches Outcome.fixture exactly (the path printed in a FAIL line, e.g.
 * "contracts/fixtures/download.pause.json"); `transport`, if given, narrows to one
 * transport. This is a maintained allowlist, not a captured snapshot: an entry that no
 * longer fails is a bug in the list, not a pass, so `applyXfail` turns that back into a
 * failure rather than silently dropping the entry. That is what keeps the list shrinking
 * as DAEMON lands handlers instead of quietly becoming a list nobody rechecks.
 */
interface XfailEntry {
  fixture: string;
  transport?: TransportName;
  reason: string;
}

function loadXfail(path: string): XfailEntry[] {
  const parsed = JSON.parse(readFileSync(path, 'utf8')) as unknown;
  if (!Array.isArray(parsed)) throw new Error(`${path}: expected a JSON array`);
  return parsed as XfailEntry[];
}

/**
 * Reconciles outcomes against the allowlist. A listed fixture that failed is downgraded
 * to a pass (its detail says why). A listed fixture that *passed* is flipped to a
 * failure: the entry is stale and must be deleted from the list, not left to rot.
 *
 * Never touches a 'static' outcome: those validate the golden fixture against the
 * generated validators offline and never talk to a server, so a stub handler can't make
 * one fail in the first place — matching them here would just relabel an
 * always-true check as "xfail" and then, since it always stays true, immediately flag it
 * as an unexpected pass. (A fixture also gets *two* static outcomes — params and result —
 * so without this exclusion a single xfail entry would print that "duplicate" twice.)
 */
function applyXfail(results: readonly Outcome[], xfail: readonly XfailEntry[]): Outcome[] {
  const matches = (e: XfailEntry, r: Outcome): boolean =>
    r.transport !== 'static' && e.fixture === r.fixture &&
    (e.transport === undefined || e.transport === r.transport);

  return results.map((r) => {
    const entry = xfail.find((e) => matches(e, r));
    if (!entry) return r;
    if (!r.ok) {
      return { ...r, ok: true, detail: `xfail (${entry.reason}): ${r.detail}` };
    }
    return {
      ...r, ok: false,
      detail: `xfail entry unexpectedly passed — delete it from the allowlist ` +
              `(was: ${entry.reason})`,
    };
  });
}

// ---------------------------------------------------------------------- main

async function main(): Promise<void> {
  const argv = process.argv.slice(2);
  const arg = (name: string): string | undefined => {
    const i = argv.indexOf(name);
    return i === -1 ? undefined : argv[i + 1];
  };

  // --only narrows the run to fixtures whose path contains a substring, and
  // --include-requires replays the ones needing a condition the harness has arranged
  // (a slow daemon, a hostile origin server). run.sh uses both to prove capture.offer
  // fails open, which cannot be shown against a healthy server.
  const only = arg('--only');
  const includeRequires = argv.includes('--include-requires');
  const all = loadFixtures();
  const fixtures = only === undefined ? all : all.filter((f) => f.file.includes(only));
  if (fixtures.length === 0) {
    process.stderr.write(`conformance: --only ${String(only)} matched no fixtures\n`);
    process.exit(2);
  }
  const results: Outcome[] = [...staticChecks(fixtures)];

  const udsPath = arg('--uds');
  const wsPort = arg('--ws-port');

  // Each opens (and re-opens, via `reconnect`) with the same handshake: session.hello on
  // the Unix socket, session.pair + session.hello on the WebSocket. Needed because at
  // least one fixture (session.hello.version-mismatch) documents that the *server* closes
  // the connection after replying — replay() calls this to get a working connection back
  // rather than leaving every later fixture on the shared connection to time out.
  async function freshUds(): Promise<Conn> {
    const conn = await connectUds(udsPath!);
    await conn.call('session.hello',
      { clientType: 'test', clientName: 'conformance', protocolVersion: '1.0.0' });
    return conn;
  }
  async function freshWs(): Promise<Conn> {
    const conn = await connectWs(Number(wsPort));
    const paired = await conn.request(
      'session.pair',
      { clientName: 'conformance', extensionId: '11111111-2222-3333-4444-555555555555' },
      5000,
    );
    const token = (paired?.result as { token?: string } | undefined)?.token;
    if (token === undefined) throw new Error('pairing failed: no token issued');
    await conn.request('session.hello',
      { clientType: 'test', clientName: 'conformance', protocolVersion: '1.0.0', token }, 5000);
    return conn;
  }

  if (udsPath) {
    const conn = await freshUds();
    const { bindings, setup } = await setupBindings(conn);
    const { outcomes, conn: last } = await replay(conn, fixtures, bindings, includeRequires, freshUds);
    results.push(...setup, ...outcomes);
    last.close();
  }
  if (wsPort) {
    const conn = await freshWs();
    const { bindings, setup } = await setupBindings(conn);
    const { outcomes, conn: last } = await replay(conn, fixtures, bindings, includeRequires, freshWs);
    results.push(...setup, ...outcomes);
    results.push(...(await privilegeChecks(last)));
    last.close();
  }
  if (!udsPath && !wsPort) {
    process.stdout.write('no --uds or --ws-port given: ran static checks only\n');
  }

  const xfailPath = arg('--xfail');
  const finalResults = xfailPath ? applyXfail(results, loadXfail(xfailPath)) : results;

  const failed = finalResults.filter((r) => !r.ok);
  for (const r of failed) {
    process.stdout.write(`FAIL [${r.transport}] ${r.fixture}\n       ${r.detail}\n`);
  }
  const byTransport = new Map<string, number>();
  for (const r of finalResults) byTransport.set(r.transport, (byTransport.get(r.transport) ?? 0) + 1);
  const summary = [...byTransport].map(([k, v]) => `${k}:${v}`).join('  ');
  process.stdout.write(
    `\n${finalResults.length - failed.length}/${finalResults.length} checks passed  (${summary})\n`);
  process.exit(failed.length === 0 ? 0 : 1);
}

main().catch((err: unknown) => {
  process.stderr.write(`conformance: ${String(err)}\n`);
  process.exit(2);
});