ext: capture/headers.ts — request-header stash
onHeadersReceived (where shouldCapture runs) doesn't carry the headers the
browser actually sent; signed-URL and referrer-gated CDNs need them. Stash
from onBeforeSendHeaders keyed by requestId, read back at capture time.
This map sees every request, so it is bounded both ways — oldest-out past a
size cap (default 2048) and a 5-minute TTL, swept on a 60 s timer and on read
— and attach() clears an entry the moment its request completes or errors.
normalizeHeaders(): Array<{name,value}> -> lower-cased map, repeats joined
with ", ". 13 tests: eviction order, redirect re-put refresh, TTL expiry,
sweep, and the onBeforeSendHeaders/onCompleted/onErrorOccurred wiring.
Co-Authored-By: Claude Sonnet 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_012Y9RU58hD1BuwP82DySUHk
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,205 @@
|
||||
// Request-header stash.
|
||||
//
|
||||
// `onHeadersReceived` (where shouldCapture runs) does not carry the request headers the
|
||||
// browser actually sent — signed-URL and referrer-gated CDNs need those. So we stash them
|
||||
// from `onBeforeSendHeaders`, keyed by requestId, and the capture hook reads them back a
|
||||
// few milliseconds later.
|
||||
//
|
||||
// This map sees every request the browser makes. An unbounded or un-expired one is a
|
||||
// memory leak that grows for the life of the session, so it is BOTH size-capped (oldest
|
||||
// out first) AND time-capped (5-minute TTL), and it is cleared for a request as soon as
|
||||
// that request finishes or errors.
|
||||
|
||||
import type { Headers } from '../../shared/protocol/index.js';
|
||||
|
||||
export interface StashedRequest {
|
||||
requestId: string;
|
||||
url: string;
|
||||
method: string;
|
||||
/** Lower-cased header names; multiple values for one name joined with ", ". */
|
||||
headers: Headers;
|
||||
tabId: number;
|
||||
stashedAt: number;
|
||||
}
|
||||
|
||||
interface HeaderEntry {
|
||||
name: string;
|
||||
value?: string;
|
||||
}
|
||||
|
||||
export interface OnBeforeSendHeadersDetails {
|
||||
requestId: string;
|
||||
url: string;
|
||||
method: string;
|
||||
tabId: number;
|
||||
requestHeaders?: HeaderEntry[];
|
||||
}
|
||||
|
||||
interface RequestEndDetails {
|
||||
requestId: string;
|
||||
}
|
||||
|
||||
interface WebRequestEvent<L extends (...args: never[]) => void> {
|
||||
addListener(listener: L, filter: { urls: string[] }, extraInfoSpec?: string[]): void;
|
||||
removeListener(listener: L): void;
|
||||
}
|
||||
|
||||
/** The slice of `browser.webRequest` this stash wires itself to. */
|
||||
export interface WebRequestLike {
|
||||
onBeforeSendHeaders: WebRequestEvent<(d: OnBeforeSendHeadersDetails) => void>;
|
||||
onCompleted: WebRequestEvent<(d: RequestEndDetails) => void>;
|
||||
onErrorOccurred: WebRequestEvent<(d: RequestEndDetails) => void>;
|
||||
}
|
||||
|
||||
export interface HeaderStashOptions {
|
||||
/** Hard ceiling on live entries; the oldest is evicted past this. Default 2048. */
|
||||
maxEntries?: number;
|
||||
/** Entries older than this are treated as absent and swept. Default 5 min. */
|
||||
ttlMs?: number;
|
||||
/** Background sweep cadence. 0 disables the timer (callers can sweep() by hand). Default 60 s. */
|
||||
sweepIntervalMs?: number;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
const ALL_URLS = '<all_urls>';
|
||||
|
||||
export class HeaderStash {
|
||||
private readonly maxEntries: number;
|
||||
private readonly ttlMs: number;
|
||||
private readonly sweepIntervalMs: number;
|
||||
private readonly now: () => number;
|
||||
|
||||
/** Insertion-ordered, so the first key is always the oldest. */
|
||||
private readonly entries = new Map<string, StashedRequest>();
|
||||
private sweepTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
private boundBeforeSend?: (d: OnBeforeSendHeadersDetails) => void;
|
||||
private boundEnd?: (d: RequestEndDetails) => void;
|
||||
private attachedTo?: WebRequestLike;
|
||||
|
||||
constructor(opts: HeaderStashOptions = {}) {
|
||||
this.maxEntries = opts.maxEntries ?? 2048;
|
||||
this.ttlMs = opts.ttlMs ?? 5 * 60_000;
|
||||
this.sweepIntervalMs = opts.sweepIntervalMs ?? 60_000;
|
||||
this.now = opts.now ?? Date.now;
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.entries.size;
|
||||
}
|
||||
|
||||
put(details: OnBeforeSendHeadersDetails): void {
|
||||
const record: StashedRequest = {
|
||||
requestId: details.requestId,
|
||||
url: details.url,
|
||||
method: details.method,
|
||||
headers: normalizeHeaders(details.requestHeaders),
|
||||
tabId: details.tabId,
|
||||
stashedAt: this.now(),
|
||||
};
|
||||
// Re-insert so a redirected request (same id, fresh onBeforeSendHeaders) moves to the
|
||||
// newest slot and its TTL restarts.
|
||||
this.entries.delete(record.requestId);
|
||||
this.entries.set(record.requestId, record);
|
||||
|
||||
while (this.entries.size > this.maxEntries) {
|
||||
const oldest = this.entries.keys().next().value;
|
||||
if (oldest === undefined) break;
|
||||
this.entries.delete(oldest);
|
||||
}
|
||||
}
|
||||
|
||||
/** Read and remove — the normal path once a request has been offered to the daemon. */
|
||||
take(requestId: string): StashedRequest | undefined {
|
||||
const found = this.peek(requestId);
|
||||
if (found) this.entries.delete(requestId);
|
||||
return found;
|
||||
}
|
||||
|
||||
/** Read without removing. Returns undefined for an unknown or expired entry (which it
|
||||
* also deletes). */
|
||||
peek(requestId: string): StashedRequest | undefined {
|
||||
const record = this.entries.get(requestId);
|
||||
if (!record) return undefined;
|
||||
if (this.now() - record.stashedAt > this.ttlMs) {
|
||||
this.entries.delete(requestId);
|
||||
return undefined;
|
||||
}
|
||||
return record;
|
||||
}
|
||||
|
||||
drop(requestId: string): void {
|
||||
this.entries.delete(requestId);
|
||||
}
|
||||
|
||||
/** Remove every expired entry. Returns how many went. */
|
||||
sweep(): number {
|
||||
const cutoff = this.now() - this.ttlMs;
|
||||
let removed = 0;
|
||||
for (const [id, record] of this.entries) {
|
||||
if (record.stashedAt <= cutoff) {
|
||||
this.entries.delete(id);
|
||||
removed += 1;
|
||||
} else {
|
||||
// insertion order == age order, so the first survivor ends the sweep
|
||||
break;
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.entries.clear();
|
||||
}
|
||||
|
||||
/** Wire onto `browser.webRequest`: stash on send, forget on finish/error, sweep on a timer. */
|
||||
attach(webRequest: WebRequestLike): void {
|
||||
if (this.attachedTo) throw new Error('HeaderStash is already attached');
|
||||
this.attachedTo = webRequest;
|
||||
|
||||
this.boundBeforeSend = (d) => this.put(d);
|
||||
this.boundEnd = (d) => this.drop(d.requestId);
|
||||
|
||||
webRequest.onBeforeSendHeaders.addListener(this.boundBeforeSend, { urls: [ALL_URLS] }, [
|
||||
'requestHeaders',
|
||||
]);
|
||||
webRequest.onCompleted.addListener(this.boundEnd, { urls: [ALL_URLS] });
|
||||
webRequest.onErrorOccurred.addListener(this.boundEnd, { urls: [ALL_URLS] });
|
||||
|
||||
if (this.sweepIntervalMs > 0) {
|
||||
this.sweepTimer = setInterval(() => this.sweep(), this.sweepIntervalMs);
|
||||
(this.sweepTimer as { unref?: () => void }).unref?.();
|
||||
}
|
||||
}
|
||||
|
||||
detach(): void {
|
||||
const webRequest = this.attachedTo;
|
||||
if (webRequest) {
|
||||
if (this.boundBeforeSend) webRequest.onBeforeSendHeaders.removeListener(this.boundBeforeSend);
|
||||
if (this.boundEnd) {
|
||||
webRequest.onCompleted.removeListener(this.boundEnd);
|
||||
webRequest.onErrorOccurred.removeListener(this.boundEnd);
|
||||
}
|
||||
}
|
||||
this.attachedTo = undefined;
|
||||
this.boundBeforeSend = undefined;
|
||||
this.boundEnd = undefined;
|
||||
if (this.sweepTimer) {
|
||||
clearInterval(this.sweepTimer);
|
||||
this.sweepTimer = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Array<{name,value}> → { "lower-name": "value[, value]" }. */
|
||||
export function normalizeHeaders(list: HeaderEntry[] | undefined): Headers {
|
||||
const out: Headers = {};
|
||||
if (!list) return out;
|
||||
for (const { name, value } of list) {
|
||||
if (value === undefined) continue;
|
||||
const key = name.toLowerCase();
|
||||
const existing = out[key];
|
||||
out[key] = existing === undefined ? value : `${existing}, ${value}`;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import {
|
||||
HeaderStash,
|
||||
normalizeHeaders,
|
||||
type OnBeforeSendHeadersDetails,
|
||||
type WebRequestLike,
|
||||
} from '../../src/background/capture/headers.js';
|
||||
|
||||
function details(over: Partial<OnBeforeSendHeadersDetails> = {}): OnBeforeSendHeadersDetails {
|
||||
return {
|
||||
requestId: '1',
|
||||
url: 'https://cdn.example.com/big.zip',
|
||||
method: 'GET',
|
||||
tabId: 7,
|
||||
requestHeaders: [
|
||||
{ name: 'User-Agent', value: 'Firefox' },
|
||||
{ name: 'Referer', value: 'https://example.com/page' },
|
||||
{ name: 'Cookie', value: 'sid=abc' },
|
||||
],
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
describe('normalizeHeaders', () => {
|
||||
it('lower-cases names and drops valueless entries', () => {
|
||||
expect(
|
||||
normalizeHeaders([
|
||||
{ name: 'Accept-Encoding', value: 'gzip' },
|
||||
{ name: 'X-Empty' },
|
||||
]),
|
||||
).toEqual({ 'accept-encoding': 'gzip' });
|
||||
});
|
||||
|
||||
it('joins repeated header names with ", "', () => {
|
||||
expect(
|
||||
normalizeHeaders([
|
||||
{ name: 'X-Fwd', value: 'a' },
|
||||
{ name: 'x-fwd', value: 'b' },
|
||||
]),
|
||||
).toEqual({ 'x-fwd': 'a, b' });
|
||||
});
|
||||
|
||||
it('returns {} for missing headers', () => {
|
||||
expect(normalizeHeaders(undefined)).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe('HeaderStash', () => {
|
||||
let now = 1_000_000;
|
||||
const clock = () => now;
|
||||
|
||||
afterEach(() => {
|
||||
now = 1_000_000;
|
||||
});
|
||||
|
||||
it('stashes and reads back normalized headers', () => {
|
||||
const s = new HeaderStash({ now: clock, sweepIntervalMs: 0 });
|
||||
s.put(details());
|
||||
const got = s.peek('1');
|
||||
expect(got).toMatchObject({
|
||||
requestId: '1',
|
||||
method: 'GET',
|
||||
tabId: 7,
|
||||
headers: { 'user-agent': 'Firefox', referer: 'https://example.com/page', cookie: 'sid=abc' },
|
||||
});
|
||||
});
|
||||
|
||||
it('take() reads once then the entry is gone', () => {
|
||||
const s = new HeaderStash({ now: clock, sweepIntervalMs: 0 });
|
||||
s.put(details());
|
||||
expect(s.take('1')).toBeDefined();
|
||||
expect(s.take('1')).toBeUndefined();
|
||||
expect(s.size).toBe(0);
|
||||
});
|
||||
|
||||
it('evicts the oldest entry past the size cap', () => {
|
||||
const s = new HeaderStash({ now: clock, maxEntries: 2, sweepIntervalMs: 0 });
|
||||
s.put(details({ requestId: 'a' }));
|
||||
s.put(details({ requestId: 'b' }));
|
||||
s.put(details({ requestId: 'c' }));
|
||||
expect(s.size).toBe(2);
|
||||
expect(s.peek('a')).toBeUndefined();
|
||||
expect(s.peek('b')).toBeDefined();
|
||||
expect(s.peek('c')).toBeDefined();
|
||||
});
|
||||
|
||||
it('re-putting a requestId refreshes its recency and TTL (redirect case)', () => {
|
||||
const s = new HeaderStash({ now: clock, maxEntries: 2, ttlMs: 1000, sweepIntervalMs: 0 });
|
||||
s.put(details({ requestId: 'a' }));
|
||||
now += 10;
|
||||
s.put(details({ requestId: 'b' }));
|
||||
now += 10;
|
||||
s.put(details({ requestId: 'a', url: 'https://cdn.example.com/redirected.zip' })); // a moves to newest
|
||||
s.put(details({ requestId: 'c' })); // evicts the now-oldest, which is b
|
||||
expect(s.peek('b')).toBeUndefined();
|
||||
expect(s.peek('a')?.url).toBe('https://cdn.example.com/redirected.zip');
|
||||
now += 995; // < 1000 since the re-put of a
|
||||
expect(s.peek('a')).toBeDefined();
|
||||
});
|
||||
|
||||
it('treats an entry past its TTL as absent and deletes it on read', () => {
|
||||
const s = new HeaderStash({ now: clock, ttlMs: 5 * 60_000, sweepIntervalMs: 0 });
|
||||
s.put(details());
|
||||
now += 5 * 60_000 + 1;
|
||||
expect(s.peek('1')).toBeUndefined();
|
||||
expect(s.size).toBe(0);
|
||||
});
|
||||
|
||||
it('sweep() drops expired entries and keeps fresh ones', () => {
|
||||
const s = new HeaderStash({ now: clock, ttlMs: 1000, sweepIntervalMs: 0 });
|
||||
s.put(details({ requestId: 'old1' }));
|
||||
s.put(details({ requestId: 'old2' }));
|
||||
now += 1001;
|
||||
s.put(details({ requestId: 'fresh' }));
|
||||
expect(s.sweep()).toBe(2);
|
||||
expect(s.size).toBe(1);
|
||||
expect(s.peek('fresh')).toBeDefined();
|
||||
});
|
||||
|
||||
it('drop() forgets one request', () => {
|
||||
const s = new HeaderStash({ now: clock, sweepIntervalMs: 0 });
|
||||
s.put(details({ requestId: 'x' }));
|
||||
s.drop('x');
|
||||
expect(s.peek('x')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// --- attach() wiring ---------------------------------------------------------------
|
||||
|
||||
class FakeEvent<L extends (...a: never[]) => void> {
|
||||
listeners: L[] = [];
|
||||
addListener = (l: L): void => void this.listeners.push(l);
|
||||
removeListener = (l: L): void => {
|
||||
this.listeners = this.listeners.filter((x) => x !== l);
|
||||
};
|
||||
emit(...args: Parameters<L>): void {
|
||||
for (const l of this.listeners) (l as (...a: Parameters<L>) => void)(...args);
|
||||
}
|
||||
}
|
||||
|
||||
function fakeWebRequest() {
|
||||
return {
|
||||
onBeforeSendHeaders: new FakeEvent<(d: OnBeforeSendHeadersDetails) => void>(),
|
||||
onCompleted: new FakeEvent<(d: { requestId: string }) => void>(),
|
||||
onErrorOccurred: new FakeEvent<(d: { requestId: string }) => void>(),
|
||||
};
|
||||
}
|
||||
|
||||
describe('HeaderStash.attach', () => {
|
||||
it('stashes on onBeforeSendHeaders and forgets on onCompleted / onErrorOccurred', () => {
|
||||
const wr = fakeWebRequest();
|
||||
const s = new HeaderStash({ sweepIntervalMs: 0 });
|
||||
s.attach(wr as unknown as WebRequestLike);
|
||||
|
||||
wr.onBeforeSendHeaders.emit(details({ requestId: 'done' }));
|
||||
wr.onBeforeSendHeaders.emit(details({ requestId: 'failed' }));
|
||||
expect(s.size).toBe(2);
|
||||
|
||||
wr.onCompleted.emit({ requestId: 'done' });
|
||||
wr.onErrorOccurred.emit({ requestId: 'failed' });
|
||||
expect(s.size).toBe(0);
|
||||
|
||||
s.detach();
|
||||
wr.onBeforeSendHeaders.emit(details({ requestId: 'after-detach' }));
|
||||
expect(s.size).toBe(0);
|
||||
expect(wr.onBeforeSendHeaders.listeners).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('runs a background sweep on its interval', () => {
|
||||
vi.useFakeTimers();
|
||||
let t = 0;
|
||||
const s = new HeaderStash({ now: () => t, ttlMs: 1000, sweepIntervalMs: 500 });
|
||||
const wr = fakeWebRequest();
|
||||
s.attach(wr as unknown as WebRequestLike);
|
||||
wr.onBeforeSendHeaders.emit(details({ requestId: 'a' }));
|
||||
|
||||
t = 2000;
|
||||
vi.advanceTimersByTime(500);
|
||||
expect(s.size).toBe(0);
|
||||
|
||||
s.detach();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('refuses a double attach', () => {
|
||||
const s = new HeaderStash({ sweepIntervalMs: 0 });
|
||||
const wr = fakeWebRequest();
|
||||
s.attach(wr as unknown as WebRequestLike);
|
||||
expect(() => s.attach(wr as unknown as WebRequestLike)).toThrow(/already attached/);
|
||||
s.detach();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user