vdm/tests/conformance/run.sh

#!/usr/bin/env bash
#
# The conformance suite. This is the command CI runs on every lane's PR.
#
#   ./tests/conformance/run.sh              static + C++ + TS against mockd, then veloxd
#   ./tests/conformance/run.sh --uds PATH --ws-port N   against an already-running daemon
#
# Four runners, one set of fixtures:
#   1. check_contract.py  schemas, fixtures and committed generated code agree
#   2. cpp/                the generated C++ parses, serialises and dispatches every fixture
#   3. ts/replay.ts        against mockd — mockd always answers every fixture correctly, so
#                          this is the TS client and the fixtures agreeing with each other
#   3b. ts/replay.ts        against a real, isolated veloxd it builds and starts — the one
#                          runner that can catch veloxd disagreeing with its own contract.
#                          Fixtures that hit a still-stubbed handler (daemon/docs/deferrals.md
#                          D1-D4b) are excused via veloxd-xfail.json; everything else must
#                          pass for real.
#
# Plus one scenario that cannot be shown against a healthy server: with the daemon
# answering slower than capture.offer's 750 ms deadline, the client must give up and let
# Firefox take the download. That is the fail-open guarantee, and it is checked here.

set -euo pipefail

REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
HERE="$REPO/tests/conformance"
WORK="$(mktemp -d)"
EXTERNAL_UDS=""
EXTERNAL_WS=""
MOCKD_PID=""
SLOW_PID=""
VELOXD_PID=""

while [ $# -gt 0 ]; do
  case "$1" in
    --uds) EXTERNAL_UDS="$2"; shift 2 ;;
    --ws-port) EXTERNAL_WS="$2"; shift 2 ;;
    -h|--help) sed -n '2,20p' "$0"; exit 0 ;;
    *) echo "run.sh: unknown option $1" >&2; exit 2 ;;
  esac
done

# Kill the server and anything it spawned. `pkill -P "$pid"` only reaps direct children —
# tsx's actual listener is often a grandchild, which that missed, leaving it holding the
# port and breaking the next run (a leaked mockd once did exactly this). Every server
# below is launched via `setsid`, which makes it the leader of its own new session/process
# group (pgid == its own pid), so `kill -TERM -"$pid"` (negative: a process-group kill)
# reaches it and everything it spawned in one shot, however deep.
stop() {
  local pid="$1"
  [ -n "$pid" ] || return 0
  kill -TERM -"$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true
  wait "$pid" 2>/dev/null || true
}

# A free loopback TCP port, kernel-assigned (bind :0) rather than a fixed number — a
# hardcoded port means one leaked process from a previous run makes every future run fail
# EADDRINUSE instead of just picking a different port.
free_port() {
  python3 -c "import socket; s=socket.socket(); s.bind(('127.0.0.1',0)); print(s.getsockname()[1]); s.close()"
}

cleanup() {
  stop "$MOCKD_PID"
  stop "$SLOW_PID"
  stop "$VELOXD_PID"
  rm -rf "$WORK"
}
trap cleanup EXIT

step() { printf '\n=== %s ===\n' "$1"; }

# ---------------------------------------------------------------- 1. static
step "static conformance (schemas, fixtures, generated code)"
# jsonschema/referencing aren't part of tools/bootstrap.sh's apt list (that's PKG's
# script; these are this suite's own Python deps), so this suite installs them itself
# rather than assuming a CI image happens to have them. Cheap and idempotent when
# they're already present, which is every local dev run after the first.
python3 -c "import jsonschema, referencing" 2>/dev/null \
  || python3 -m pip install --quiet --disable-pip-version-check --user jsonschema referencing
python3 "$HERE/check_contract.py"

# ------------------------------------------------------------------- 2. C++
step "generated C++ (parse, serialise, dispatch)"
CXX="${CXX:-g++}"
"$CXX" -std=c++23 -Wall -Wextra -Wpedantic -Werror \
  -I"$REPO/core/generated" -I"$HERE/cpp" \
  "$HERE/cpp/conformance_main.cpp" "$REPO/core/generated/velox_proto.cpp" \
  -o "$WORK/conformance_cpp"
"$WORK/conformance_cpp" "$REPO"

# -------------------------------------------------------------------- 3. TS
step "generated TypeScript against a live server"
if [ -z "$EXTERNAL_UDS" ] && [ -z "$EXTERNAL_WS" ]; then
  ( cd "$REPO/tools/mockd" && npm install --silent --no-audit --no-fund )
  UDS="$WORK/velox.sock"
  WS_PORT="$(free_port)"
  ( cd "$REPO/tools/mockd" && exec setsid ./node_modules/.bin/tsx src/index.ts \
      --uds "$UDS" --ws-port "$WS_PORT" --allowed-root "$WORK" ) >"$WORK/mockd.log" 2>&1 &
  MOCKD_PID=$!
  # Wait for the socket rather than sleeping a guessed amount.
  for _ in $(seq 1 50); do
    [ -S "$UDS" ] && node -e "require('net').connect('$UDS').on('connect',function(){this.end();process.exit(0)}).on('error',()=>process.exit(1))" 2>/dev/null && break
    sleep 0.2
  done
  node -e "require('net').connect('$UDS').on('connect',function(){this.end();process.exit(0)}).on('error',()=>process.exit(1))" 2>/dev/null \
    || { echo "mockd did not start:"; cat "$WORK/mockd.log"; exit 1; }
else
  UDS="$EXTERNAL_UDS"
  WS_PORT="$EXTERNAL_WS"
fi

( cd "$HERE/ts" && npm install --silent --no-audit --no-fund )

TS_ARGS=()
[ -n "$UDS" ] && TS_ARGS+=(--uds "$UDS")
[ -n "$WS_PORT" ] && TS_ARGS+=(--ws-port "$WS_PORT")
( cd "$HERE/ts" && ./node_modules/.bin/tsx replay.ts "${TS_ARGS[@]}" )

# ------------------------------------------------------------------ 3b. veloxd
# The same fixtures against a real, isolated veloxd. mockd (above) always answers every
# fixture correctly by construction, so it can only prove the TS client and the fixtures
# agree with each other — it cannot catch veloxd disagreeing with its own contract. This
# is the runner that closed that gap: it is what would have caught veloxd's download.get
# returning `segments: 0`, which TaskSummary forbids (minimum 1, required), before it
# shipped rather than after.
step "generated TypeScript against a real, isolated veloxd"
if [ -z "$EXTERNAL_UDS" ] && [ -z "$EXTERNAL_WS" ]; then
  if [ ! -f "$REPO/daemon/CMakeLists.txt" ]; then
    echo "skipped: daemon/CMakeLists.txt not present (lane DAEMON has not landed yet)"
  else
    VBUILD="$REPO/build/dev"
    # cmake --preset dev is idempotent to re-run against an existing build dir; a CI leg
    # that already configured (the `conformance` job does, before ctest) just reuses it.
    if [ ! -f "$VBUILD/CMakeCache.txt" ]; then
      ( cd "$REPO" && cmake --preset dev ) >"$WORK/veloxd-configure.log" 2>&1 \
        || { echo "veloxd: cmake configure failed:"; cat "$WORK/veloxd-configure.log"; exit 1; }
    fi
    cmake --build "$VBUILD" --target veloxd >"$WORK/veloxd-build.log" 2>&1 \
      || { echo "veloxd: build failed:"; cat "$WORK/veloxd-build.log"; exit 1; }
    VELOXD_BIN="$VBUILD/bin/veloxd"

    # Isolated: its own runtime dir (socket, ws.port, single-instance lock), data dir
    # (velox.db) and config dir, none of them the real user's. veloxd's single-instance
    # lock is a UID-scoped abstract socket, not namespaced by XDG_RUNTIME_DIR, so this
    # still collides with a veloxd already running for this user outside the sandbox —
    # that shows up below as "another instance is already running" and fails loudly
    # rather than silently testing the wrong daemon.
    VXDG="$WORK/veloxd-xdg"
    mkdir -p "$VXDG/runtime" "$VXDG/data" "$VXDG/config" "$VXDG/downloads"

    # VELOX_PAIR_AUTO=1: the pairing approver is the D1 dev stub (EnvAutoApprover,
    # daemon/src/rpc/pairing.cpp) and denies every pairing without it — without this,
    # session.pair never issues a token and the WS half of this step can't even connect.
    XDG_RUNTIME_DIR="$VXDG/runtime" XDG_DATA_HOME="$VXDG/data" XDG_CONFIG_HOME="$VXDG/config" \
      VELOX_PAIR_AUTO=1 \
      setsid "$VELOXD_BIN" >"$WORK/veloxd.log" 2>&1 &
    VELOXD_PID=$!
    VUDS="$VXDG/runtime/velox/velox.sock"
    for _ in $(seq 1 50); do [ -S "$VUDS" ] && break; sleep 0.2; done
    [ -S "$VUDS" ] || { echo "veloxd did not start:"; cat "$WORK/veloxd.log"; exit 1; }

    # saveTo.allowedRoots defaults to ["~/Downloads"]; download.add's own isolated
    # downloads dir needs to be an allowed root too, or every download.add fixture fails
    # -32011 before the point of this runner is even reached. $HOME/Downloads stays in
    # the list alongside it: a few fixtures still set an explicit saveDir there
    # (category.upsert.json, download.update.json) rather than take the default.
    # capture.minSizeBytes defaults to 0 (nothing is ever "too small"), which makes
    # errors/capture.offer.ignore.json's below-minimum-size case impossible to reach
    # against a fresh daemon; raised here so that fixture's scenario is actually
    # reachable. Written straight into the isolated velox.db, before veloxd has any RPC
    # session to write it through: settings.set is real now (D9), but this has to be in
    # place before the very first fixture runs, and setup happens before any connection
    # exists.
    python3 - "$VXDG/data/velox/velox.db" "$VXDG/downloads" "$HOME/Downloads" <<'PY'
import json, sqlite3, sys
db_path, isolated_downloads, home_downloads = sys.argv[1:4]
db = sqlite3.connect(db_path)
db.execute(
    "INSERT INTO settings(key, value) VALUES(?, ?) "
    "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
    ("saveTo.allowedRoots", json.dumps([isolated_downloads, home_downloads])),
)
db.execute(
    "INSERT INTO settings(key, value) VALUES(?, ?) "
    "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
    ("capture.minSizeBytes", json.dumps(1000000)),
)
db.execute(
    "INSERT INTO settings(key, value) VALUES(?, ?) "
    "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
    ("saveTo.defaultDir", json.dumps(isolated_downloads)),
)
db.commit()
PY

    VELOXD_TS_ARGS=(--uds "$VUDS")
    if [ -f "$VXDG/runtime/velox/ws.port" ]; then
      VELOXD_TS_ARGS+=(--ws-port "$(cat "$VXDG/runtime/velox/ws.port")")
    fi
    ( cd "$HERE/ts" && ./node_modules/.bin/tsx replay.ts "${VELOXD_TS_ARGS[@]}" \
        --xfail "$HERE/veloxd-xfail.json" )
  fi
else
  echo "skipped: --uds/--ws-port already points at a live daemon"
fi

# ------------------------------------------------- 4. capture fails open
step "capture.offer fails open when the daemon is too slow"
if [ -z "$EXTERNAL_UDS" ]; then
  SLOW_UDS="$WORK/slow.sock"
  ( cd "$REPO/tools/mockd" && exec setsid ./node_modules/.bin/tsx src/index.ts \
      --uds "$SLOW_UDS" --no-ws --slow 2000 ) >"$WORK/slow.log" 2>&1 &
  SLOW_PID=$!
  for _ in $(seq 1 50); do [ -S "$SLOW_UDS" ] && break; sleep 0.2; done
  [ -S "$SLOW_UDS" ] || { echo "slow mockd did not start:"; cat "$WORK/slow.log"; exit 1; }
  ( cd "$HERE/ts" && ./node_modules/.bin/tsx replay.ts --uds "$SLOW_UDS" \
      --only capture.offer.timeout --include-requires )
else
  echo "skipped: needs a deliberately slow server, which run.sh only arranges for mockd"
fi

printf '\n=== conformance: all runners passed ===\n'