ext: capture/headers.ts — request-header stash

onHeadersReceived (where shouldCapture runs) doesn't carry the headers the
browser actually sent; signed-URL and referrer-gated CDNs need them. Stash
from onBeforeSendHeaders keyed by requestId, read back at capture time.

This map sees every request, so it is bounded both ways — oldest-out past a
size cap (default 2048) and a 5-minute TTL, swept on a 60 s timer and on read
— and attach() clears an entry the moment its request completes or errors.

normalizeHeaders(): Array<{name,value}> -> lower-cased map, repeats joined
with ", ". 13 tests: eviction order, redirect re-put refresh, TTL expiry,
sweep, and the onBeforeSendHeaders/onCompleted/onErrorOccurred wiring.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_012Y9RU58hD1BuwP82DySUHk

extension/src/background/capture/headers.ts

@@ -0,0 +1,205 @@
+// Request-header stash.
+//
+// `onHeadersReceived` (where shouldCapture runs) does not carry the request headers the
+// browser actually sent — signed-URL and referrer-gated CDNs need those. So we stash them
+// from `onBeforeSendHeaders`, keyed by requestId, and the capture hook reads them back a
+// few milliseconds later.
+//
+// This map sees every request the browser makes. An unbounded or un-expired one is a
+// memory leak that grows for the life of the session, so it is BOTH size-capped (oldest
+// out first) AND time-capped (5-minute TTL), and it is cleared for a request as soon as
+// that request finishes or errors.
+
+import type { Headers } from '../../shared/protocol/index.js';
+
+export interface StashedRequest {
+  requestId: string;
+  url: string;
+  method: string;
+  /** Lower-cased header names; multiple values for one name joined with ", ". */
+  headers: Headers;
+  tabId: number;
+  stashedAt: number;
+}
+
+interface HeaderEntry {
+  name: string;
+  value?: string;
+}
+
+export interface OnBeforeSendHeadersDetails {
+  requestId: string;
+  url: string;
+  method: string;
+  tabId: number;
+  requestHeaders?: HeaderEntry[];
+}
+
+interface RequestEndDetails {
+  requestId: string;
+}
+
+interface WebRequestEvent<L extends (...args: never[]) => void> {
+  addListener(listener: L, filter: { urls: string[] }, extraInfoSpec?: string[]): void;
+  removeListener(listener: L): void;
+}
+
+/** The slice of `browser.webRequest` this stash wires itself to. */
+export interface WebRequestLike {
+  onBeforeSendHeaders: WebRequestEvent<(d: OnBeforeSendHeadersDetails) => void>;
+  onCompleted: WebRequestEvent<(d: RequestEndDetails) => void>;
+  onErrorOccurred: WebRequestEvent<(d: RequestEndDetails) => void>;
+}
+
+export interface HeaderStashOptions {
+  /** Hard ceiling on live entries; the oldest is evicted past this. Default 2048. */
+  maxEntries?: number;
+  /** Entries older than this are treated as absent and swept. Default 5 min. */
+  ttlMs?: number;
+  /** Background sweep cadence. 0 disables the timer (callers can sweep() by hand). Default 60 s. */
+  sweepIntervalMs?: number;
+  now?: () => number;
+}
+
+const ALL_URLS = '<all_urls>';
+
+export class HeaderStash {
+  private readonly maxEntries: number;
+  private readonly ttlMs: number;
+  private readonly sweepIntervalMs: number;
+  private readonly now: () => number;
+
+  /** Insertion-ordered, so the first key is always the oldest. */
+  private readonly entries = new Map<string, StashedRequest>();
+  private sweepTimer: ReturnType<typeof setInterval> | null = null;
+
+  private boundBeforeSend?: (d: OnBeforeSendHeadersDetails) => void;
+  private boundEnd?: (d: RequestEndDetails) => void;
+  private attachedTo?: WebRequestLike;
+
+  constructor(opts: HeaderStashOptions = {}) {
+    this.maxEntries = opts.maxEntries ?? 2048;
+    this.ttlMs = opts.ttlMs ?? 5 * 60_000;
+    this.sweepIntervalMs = opts.sweepIntervalMs ?? 60_000;
+    this.now = opts.now ?? Date.now;
+  }
+
+  get size(): number {
+    return this.entries.size;
+  }
+
+  put(details: OnBeforeSendHeadersDetails): void {
+    const record: StashedRequest = {
+      requestId: details.requestId,
+      url: details.url,
+      method: details.method,
+      headers: normalizeHeaders(details.requestHeaders),
+      tabId: details.tabId,
+      stashedAt: this.now(),
+    };
+    // Re-insert so a redirected request (same id, fresh onBeforeSendHeaders) moves to the
+    // newest slot and its TTL restarts.
+    this.entries.delete(record.requestId);
+    this.entries.set(record.requestId, record);
+
+    while (this.entries.size > this.maxEntries) {
+      const oldest = this.entries.keys().next().value;
+      if (oldest === undefined) break;
+      this.entries.delete(oldest);
+    }
+  }
+
+  /** Read and remove — the normal path once a request has been offered to the daemon. */
+  take(requestId: string): StashedRequest | undefined {
+    const found = this.peek(requestId);
+    if (found) this.entries.delete(requestId);
+    return found;
+  }
+
+  /** Read without removing. Returns undefined for an unknown or expired entry (which it
+   *  also deletes). */
+  peek(requestId: string): StashedRequest | undefined {
+    const record = this.entries.get(requestId);
+    if (!record) return undefined;
+    if (this.now() - record.stashedAt > this.ttlMs) {
+      this.entries.delete(requestId);
+      return undefined;
+    }
+    return record;
+  }
+
+  drop(requestId: string): void {
+    this.entries.delete(requestId);
+  }
+
+  /** Remove every expired entry. Returns how many went. */
+  sweep(): number {
+    const cutoff = this.now() - this.ttlMs;
+    let removed = 0;
+    for (const [id, record] of this.entries) {
+      if (record.stashedAt <= cutoff) {
+        this.entries.delete(id);
+        removed += 1;
+      } else {
+        // insertion order == age order, so the first survivor ends the sweep
+        break;
+      }
+    }
+    return removed;
+  }
+
+  clear(): void {
+    this.entries.clear();
+  }
+
+  /** Wire onto `browser.webRequest`: stash on send, forget on finish/error, sweep on a timer. */
+  attach(webRequest: WebRequestLike): void {
+    if (this.attachedTo) throw new Error('HeaderStash is already attached');
+    this.attachedTo = webRequest;
+
+    this.boundBeforeSend = (d) => this.put(d);
+    this.boundEnd = (d) => this.drop(d.requestId);
+
+    webRequest.onBeforeSendHeaders.addListener(this.boundBeforeSend, { urls: [ALL_URLS] }, [
+      'requestHeaders',
+    ]);
+    webRequest.onCompleted.addListener(this.boundEnd, { urls: [ALL_URLS] });
+    webRequest.onErrorOccurred.addListener(this.boundEnd, { urls: [ALL_URLS] });
+
+    if (this.sweepIntervalMs > 0) {
+      this.sweepTimer = setInterval(() => this.sweep(), this.sweepIntervalMs);
+      (this.sweepTimer as { unref?: () => void }).unref?.();
+    }
+  }
+
+  detach(): void {
+    const webRequest = this.attachedTo;
+    if (webRequest) {
+      if (this.boundBeforeSend) webRequest.onBeforeSendHeaders.removeListener(this.boundBeforeSend);
+      if (this.boundEnd) {
+        webRequest.onCompleted.removeListener(this.boundEnd);
+        webRequest.onErrorOccurred.removeListener(this.boundEnd);
+      }
+    }
+    this.attachedTo = undefined;
+    this.boundBeforeSend = undefined;
+    this.boundEnd = undefined;
+    if (this.sweepTimer) {
+      clearInterval(this.sweepTimer);
+      this.sweepTimer = null;
+    }
+  }
+}
+
+/** Array<{name,value}> → { "lower-name": "value[, value]" }. */
+export function normalizeHeaders(list: HeaderEntry[] | undefined): Headers {
+  const out: Headers = {};
+  if (!list) return out;
+  for (const { name, value } of list) {
+    if (value === undefined) continue;
+    const key = name.toLowerCase();
+    const existing = out[key];
+    out[key] = existing === undefined ? value : `${existing}, ${value}`;
+  }
+  return out;
+}

extension/tests/capture/headers.test.ts

@@ -0,0 +1,193 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+import {
+  HeaderStash,
+  normalizeHeaders,
+  type OnBeforeSendHeadersDetails,
+  type WebRequestLike,
+} from '../../src/background/capture/headers.js';
+
+function details(over: Partial<OnBeforeSendHeadersDetails> = {}): OnBeforeSendHeadersDetails {
+  return {
+    requestId: '1',
+    url: 'https://cdn.example.com/big.zip',
+    method: 'GET',
+    tabId: 7,
+    requestHeaders: [
+      { name: 'User-Agent', value: 'Firefox' },
+      { name: 'Referer', value: 'https://example.com/page' },
+      { name: 'Cookie', value: 'sid=abc' },
+    ],
+    ...over,
+  };
+}
+
+describe('normalizeHeaders', () => {
+  it('lower-cases names and drops valueless entries', () => {
+    expect(
+      normalizeHeaders([
+        { name: 'Accept-Encoding', value: 'gzip' },
+        { name: 'X-Empty' },
+      ]),
+    ).toEqual({ 'accept-encoding': 'gzip' });
+  });
+
+  it('joins repeated header names with ", "', () => {
+    expect(
+      normalizeHeaders([
+        { name: 'X-Fwd', value: 'a' },
+        { name: 'x-fwd', value: 'b' },
+      ]),
+    ).toEqual({ 'x-fwd': 'a, b' });
+  });
+
+  it('returns {} for missing headers', () => {
+    expect(normalizeHeaders(undefined)).toEqual({});
+  });
+});
+
+describe('HeaderStash', () => {
+  let now = 1_000_000;
+  const clock = () => now;
+
+  afterEach(() => {
+    now = 1_000_000;
+  });
+
+  it('stashes and reads back normalized headers', () => {
+    const s = new HeaderStash({ now: clock, sweepIntervalMs: 0 });
+    s.put(details());
+    const got = s.peek('1');
+    expect(got).toMatchObject({
+      requestId: '1',
+      method: 'GET',
+      tabId: 7,
+      headers: { 'user-agent': 'Firefox', referer: 'https://example.com/page', cookie: 'sid=abc' },
+    });
+  });
+
+  it('take() reads once then the entry is gone', () => {
+    const s = new HeaderStash({ now: clock, sweepIntervalMs: 0 });
+    s.put(details());
+    expect(s.take('1')).toBeDefined();
+    expect(s.take('1')).toBeUndefined();
+    expect(s.size).toBe(0);
+  });
+
+  it('evicts the oldest entry past the size cap', () => {
+    const s = new HeaderStash({ now: clock, maxEntries: 2, sweepIntervalMs: 0 });
+    s.put(details({ requestId: 'a' }));
+    s.put(details({ requestId: 'b' }));
+    s.put(details({ requestId: 'c' }));
+    expect(s.size).toBe(2);
+    expect(s.peek('a')).toBeUndefined();
+    expect(s.peek('b')).toBeDefined();
+    expect(s.peek('c')).toBeDefined();
+  });
+
+  it('re-putting a requestId refreshes its recency and TTL (redirect case)', () => {
+    const s = new HeaderStash({ now: clock, maxEntries: 2, ttlMs: 1000, sweepIntervalMs: 0 });
+    s.put(details({ requestId: 'a' }));
+    now += 10;
+    s.put(details({ requestId: 'b' }));
+    now += 10;
+    s.put(details({ requestId: 'a', url: 'https://cdn.example.com/redirected.zip' })); // a moves to newest
+    s.put(details({ requestId: 'c' })); // evicts the now-oldest, which is b
+    expect(s.peek('b')).toBeUndefined();
+    expect(s.peek('a')?.url).toBe('https://cdn.example.com/redirected.zip');
+    now += 995; // < 1000 since the re-put of a
+    expect(s.peek('a')).toBeDefined();
+  });
+
+  it('treats an entry past its TTL as absent and deletes it on read', () => {
+    const s = new HeaderStash({ now: clock, ttlMs: 5 * 60_000, sweepIntervalMs: 0 });
+    s.put(details());
+    now += 5 * 60_000 + 1;
+    expect(s.peek('1')).toBeUndefined();
+    expect(s.size).toBe(0);
+  });
+
+  it('sweep() drops expired entries and keeps fresh ones', () => {
+    const s = new HeaderStash({ now: clock, ttlMs: 1000, sweepIntervalMs: 0 });
+    s.put(details({ requestId: 'old1' }));
+    s.put(details({ requestId: 'old2' }));
+    now += 1001;
+    s.put(details({ requestId: 'fresh' }));
+    expect(s.sweep()).toBe(2);
+    expect(s.size).toBe(1);
+    expect(s.peek('fresh')).toBeDefined();
+  });
+
+  it('drop() forgets one request', () => {
+    const s = new HeaderStash({ now: clock, sweepIntervalMs: 0 });
+    s.put(details({ requestId: 'x' }));
+    s.drop('x');
+    expect(s.peek('x')).toBeUndefined();
+  });
+});
+
+// --- attach() wiring ---------------------------------------------------------------
+
+class FakeEvent<L extends (...a: never[]) => void> {
+  listeners: L[] = [];
+  addListener = (l: L): void => void this.listeners.push(l);
+  removeListener = (l: L): void => {
+    this.listeners = this.listeners.filter((x) => x !== l);
+  };
+  emit(...args: Parameters<L>): void {
+    for (const l of this.listeners) (l as (...a: Parameters<L>) => void)(...args);
+  }
+}
+
+function fakeWebRequest() {
+  return {
+    onBeforeSendHeaders: new FakeEvent<(d: OnBeforeSendHeadersDetails) => void>(),
+    onCompleted: new FakeEvent<(d: { requestId: string }) => void>(),
+    onErrorOccurred: new FakeEvent<(d: { requestId: string }) => void>(),
+  };
+}
+
+describe('HeaderStash.attach', () => {
+  it('stashes on onBeforeSendHeaders and forgets on onCompleted / onErrorOccurred', () => {
+    const wr = fakeWebRequest();
+    const s = new HeaderStash({ sweepIntervalMs: 0 });
+    s.attach(wr as unknown as WebRequestLike);
+
+    wr.onBeforeSendHeaders.emit(details({ requestId: 'done' }));
+    wr.onBeforeSendHeaders.emit(details({ requestId: 'failed' }));
+    expect(s.size).toBe(2);
+
+    wr.onCompleted.emit({ requestId: 'done' });
+    wr.onErrorOccurred.emit({ requestId: 'failed' });
+    expect(s.size).toBe(0);
+
+    s.detach();
+    wr.onBeforeSendHeaders.emit(details({ requestId: 'after-detach' }));
+    expect(s.size).toBe(0);
+    expect(wr.onBeforeSendHeaders.listeners).toHaveLength(0);
+  });
+
+  it('runs a background sweep on its interval', () => {
+    vi.useFakeTimers();
+    let t = 0;
+    const s = new HeaderStash({ now: () => t, ttlMs: 1000, sweepIntervalMs: 500 });
+    const wr = fakeWebRequest();
+    s.attach(wr as unknown as WebRequestLike);
+    wr.onBeforeSendHeaders.emit(details({ requestId: 'a' }));
+
+    t = 2000;
+    vi.advanceTimersByTime(500);
+    expect(s.size).toBe(0);
+
+    s.detach();
+    vi.useRealTimers();
+  });
+
+  it('refuses a double attach', () => {
+    const s = new HeaderStash({ sweepIntervalMs: 0 });
+    const wr = fakeWebRequest();
+    s.attach(wr as unknown as WebRequestLike);
+    expect(() => s.attach(wr as unknown as WebRequestLike)).toThrow(/already attached/);
+    s.detach();
+  });
+});