sami/vdm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

core: net/probe + Content-Disposition parser + URL splitter (stage 3)

Browse Source 
net/content_disposition — total parser for the mojibake-prone header:
RFC 6266 filename (quoted/token), RFC 5987 filename* ext-values
(charset'lang'pct-encoded, incl. RFC 2231 continuations), legacy RFC 2047
encoded-words (=?UTF-8?B?..?= / ?Q?), and raw Latin-1 bytes; prefers
filename* over filename; strips path components AFTER decoding (a base64
payload can hold '/'). 22-case test table.

net/text_codec (internal) — percent-decode, UTF-8 validation, Latin-1->
UTF-8, base64, RFC 2047 — shared by the CD parser and the URL splitter.

net/url — a small total URL splitter (scheme/userinfo/host/port/path/
query/fragment, http(s) validity) and url_filename() for the last path
segment; used for the filename fallback.

net/probe — HEAD then a ranged GET bytes=0-0 that PROVES resumability
(206 + matching Content-Range + a validator), rather than trusting
Accept-Ranges which servers lie about; the ranged GET is also the HEAD-
refused (403/405/501) fallback. 401/407 -> success result with
requires_auth, not an error. Runs on its own pool (max_concurrent,
default 4) outside the segment budget per ADR 0011 §5. suggest_filename()
does the resolution order (explicit -> disposition -> URL -> download.bin)
with a light strip; rules/ (stage 9) owns the authoritative sanitize.

tools/fuzz — libFuzzer targets for the CD parser and the URL splitter,
compiling the parser sources directly so they're fully instrumented;
self-guards on VELOX_BUILD_FUZZ + Clang (the top-level CMake adds every
tools/* unconditionally). Seed corpora included.

Fixed on the way: a p -> Transfer -> State -> cbs -> p reference cycle in
Prober that leaked every probe (drop the stored Transfer; the worker
keeps State alive). Tests green under ASan/UBSan and TSan.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01HPPSGhiArbvQgwC2DNiURS
This commit is contained in:
sami
2026-09-09 23:53:55 +04:00
co-authored by Claude Sonnet 5
parent fdacf732fa
commit 201ebc55d4
25 changed files with 1731 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/tests/net/content_disposition_test.cpp
+148
View File
@@ -0,0 +1,148 @@
#include "vdm/net/content_disposition.hpp"
#include <string>
#include <string_view>
#include "vtest.hpp"
using vdm::net::ContentDisposition;
using vdm::net::parse_content_disposition;
using Type = vdm::net::ContentDisposition::Type;
namespace {
// "€" is U+20AC -> UTF-8 E2 82 AC. "£" is U+00A3 -> UTF-8 C2 A3.
const std::string kEuro = "\xE2\x82\xAC";
const std::string kPound = "\xC2\xA3";
const std::string kEAcute = "\xC3\xA9"; // é U+00E9
} // namespace
VT_TEST(cd_plain_quoted) {
auto cd = parse_content_disposition(R"(attachment; filename="report.pdf")");
VT_CHECK(cd.type == Type::attachment);
VT_CHECK_EQ(cd.filename, std::string("report.pdf"));
VT_CHECK(!cd.filename_from_ext);
}
VT_TEST(cd_plain_token_unquoted) {
auto cd = parse_content_disposition("attachment; filename=report.pdf");
VT_CHECK_EQ(cd.filename, std::string("report.pdf"));
}
VT_TEST(cd_inline_no_filename) {
auto cd = parse_content_disposition("inline");
VT_CHECK(cd.type == Type::inline_);
VT_CHECK(!cd.has_filename());
}
VT_TEST(cd_rfc5987_utf8_ext_value) {
auto cd = parse_content_disposition("attachment; filename*=UTF-8''%e2%82%ac%20rates.pdf");
VT_CHECK_EQ(cd.filename, kEuro + " rates.pdf");
VT_CHECK(cd.filename_from_ext);
}
VT_TEST(cd_prefers_ext_over_plain) {
auto cd = parse_content_disposition(
R"(attachment; filename="EURO rates.pdf"; filename*=UTF-8''%e2%82%ac%20rates.pdf)");
VT_CHECK_EQ(cd.filename, kEuro + " rates.pdf");
VT_CHECK(cd.filename_from_ext);
}
VT_TEST(cd_rfc5987_latin1_ext_value) {
// %A3 = £ in ISO-8859-1
auto cd = parse_content_disposition("attachment; filename*=ISO-8859-1''%A3rates.pdf");
VT_CHECK_EQ(cd.filename, kPound + "rates.pdf");
}
VT_TEST(cd_legacy_rfc2047_base64) {
// base64("<euro> rates.pdf") with euro as UTF-8
// "€ rates.pdf" -> bytes E2 82 AC 20 72 61 74 65 73 2E 70 64 66 -> base64:
auto cd =
parse_content_disposition(R"(attachment; filename="=?UTF-8?B?4oKsIHJhdGVzLnBkZg==?=")");
VT_CHECK_EQ(cd.filename, kEuro + " rates.pdf");
}
VT_TEST(cd_legacy_rfc2047_qencoded_latin1) {
// =?ISO-8859-1?Q?=A3rates.pdf?= -> £rates.pdf
auto cd = parse_content_disposition(R"(attachment; filename="=?ISO-8859-1?Q?=A3rates.pdf?=")");
VT_CHECK_EQ(cd.filename, kPound + "rates.pdf");
}
VT_TEST(cd_raw_utf8_bytes_in_quotes) {
std::string h = "attachment; filename=\"caf" + kEAcute + ".txt\"";
auto cd = parse_content_disposition(h);
VT_CHECK_EQ(cd.filename, "caf" + kEAcute + ".txt");
}
VT_TEST(cd_raw_latin1_byte_in_quotes) {
// 0xE9 is 'é' in Latin-1; not valid UTF-8 alone -> transcoded
std::string h = "attachment; filename=\"caf\xE9.txt\"";
auto cd = parse_content_disposition(h);
VT_CHECK_EQ(cd.filename, "caf" + kEAcute + ".txt");
}
VT_TEST(cd_strips_path_components) {
VT_CHECK_EQ(parse_content_disposition(R"(attachment; filename="../../etc/passwd")").filename,
std::string("passwd"));
// real Windows paths in the wild use unescaped backslashes as separators
VT_CHECK_EQ(parse_content_disposition(R"(attachment; filename="C:\Windows\evil.exe")").filename,
std::string("evil.exe"));
// a base64 payload can contain '/', so decode must happen before path stripping
VT_CHECK_EQ(parse_content_disposition(R"(attachment; filename="=?UTF-8?B?Li4vLi4vc2VjcmV0?=")")
.filename,
std::string("secret")); // decodes to "../../secret", then stripped
}
VT_TEST(cd_quoted_dquote_escape) {
// \" is the one escape we resolve, so a quote can appear mid-name
auto cd = parse_content_disposition(R"(attachment; filename="quote\"here.txt")");
VT_CHECK_EQ(cd.filename, std::string("quote\"here.txt"));
}
VT_TEST(cd_semicolon_inside_quotes_is_not_a_separator) {
auto cd = parse_content_disposition(R"(attachment; filename="a;b;c.txt")");
VT_CHECK_EQ(cd.filename, std::string("a;b;c.txt"));
}
VT_TEST(cd_form_data) {
auto cd = parse_content_disposition(R"(form-data; name="file"; filename="upload.bin")");
VT_CHECK(cd.type == Type::form_data);
VT_CHECK_EQ(cd.filename, std::string("upload.bin"));
}
VT_TEST(cd_rfc2231_continuations) {
auto cd = parse_content_disposition(
"attachment; filename*0*=UTF-8''%e2%82%ac; filename*1*=%20rates; filename*2=.pdf");
VT_CHECK_EQ(cd.filename, kEuro + " rates.pdf");
}
VT_TEST(cd_empty_and_garbage_do_not_crash) {
VT_CHECK(!parse_content_disposition("").has_filename());
VT_CHECK(!parse_content_disposition(";;;;").has_filename());
VT_CHECK(!parse_content_disposition("attachment;").has_filename());
VT_CHECK(!parse_content_disposition(R"(attachment; filename=)").has_filename());
VT_CHECK(!parse_content_disposition(R"(attachment; filename="")").has_filename());
// truncated ext-value
auto cd = parse_content_disposition("attachment; filename*=UTF-8''%e2%82");
VT_CHECK(cd.type == Type::attachment); // no crash; filename is whatever fell out
// truncated encoded-word
auto trunc = parse_content_disposition(R"(attachment; filename="=?UTF-8?B?4oKs")");
(void)trunc;
}
VT_TEST(cd_bad_percent_escapes_in_ext_value) {
// stray % and non-hex digits are emitted literally, no crash
auto cd = parse_content_disposition("attachment; filename*=UTF-8''%ZZ%%file%2");
VT_CHECK(cd.type == Type::attachment);
}
VT_TEST(cd_case_insensitive_keys_and_type) {
auto cd = parse_content_disposition(R"(ATTACHMENT; FileName="x.txt")");
VT_CHECK(cd.type == Type::attachment);
VT_CHECK_EQ(cd.filename, std::string("x.txt"));
}
VT_TEST(cd_unknown_type_is_other) {
auto cd = parse_content_disposition(R"(signal; filename="x.txt")");
VT_CHECK(cd.type == Type::other);
VT_CHECK_EQ(cd.filename, std::string("x.txt"));
}
core/tests/net/probe_test.cpp
+163
View File
@@ -0,0 +1,163 @@
#include "vdm/net/probe.hpp"
#include <chrono>
#include <future>
#include <string>
#include "testserver_fixture.hpp"
#include "vtest.hpp"
using namespace vdm;
using namespace vdm::net;
using vdm::testing::TestServer;
namespace {
Result<ProbeResult> run_probe(Prober &p, const std::string &url) {
std::promise<Result<ProbeResult>> prom;
auto fut = prom.get_future();
ProbeRequest req;
req.url = url;
req.overall_timeout_ms = 15000;
p.probe(std::move(req), [&](Result<ProbeResult> r) { prom.set_value(std::move(r)); });
if (fut.wait_for(std::chrono::seconds(25)) != std::future_status::ready)
return Err{Error::timeout, "probe test wait"};
return fut.get();
}
const std::string kEuro = "\xE2\x82\xAC";
} // namespace
VT_TEST(probe_plain_resumable_file) {
TestServer srv;
VT_REQUIRE(srv.available());
Prober p(4);
auto r = run_probe(p, srv.url("/plain/file/2M"));
VT_REQUIRE(r.has_value());
const auto &pr = r.value();
VT_CHECK_EQ(pr.http_status, 206); // proven via the ranged GET
VT_CHECK(pr.accept_ranges);
VT_CHECK(pr.resumable); // 206 + ETag validator
VT_REQUIRE(pr.total_size.has_value());
VT_CHECK_EQ(*pr.total_size, 2u * 1024 * 1024);
VT_CHECK(!pr.etag.empty());
VT_CHECK_EQ(pr.filename_from_url, std::string("2M"));
VT_CHECK(!pr.requires_auth);
}
VT_TEST(probe_no_range_server_is_not_resumable) {
TestServer srv;
VT_REQUIRE(srv.available());
Prober p(4);
auto r = run_probe(p, srv.url("/no-range/file/1M"));
VT_REQUIRE(r.has_value());
const auto &pr = r.value();
VT_CHECK(!pr.resumable); // no 206 ever
VT_CHECK(!pr.accept_ranges);
VT_REQUIRE(pr.total_size.has_value());
VT_CHECK_EQ(*pr.total_size, 1u * 1024 * 1024);
}
VT_TEST(probe_lying_accept_ranges_still_not_resumable) {
TestServer srv;
VT_REQUIRE(srv.available());
Prober p(4);
// advertises Accept-Ranges: bytes but never returns 206
auto r = run_probe(p, srv.url("/lies-about-accept-ranges/file/1M"));
VT_REQUIRE(r.has_value());
const auto &pr = r.value();
VT_CHECK(pr.accept_ranges); // it advertised
VT_CHECK(!pr.resumable); // ...but never proved it -> resume is off
}
VT_TEST(probe_reads_utf8_content_disposition) {
TestServer srv;
VT_REQUIRE(srv.available());
Prober p(4);
auto r = run_probe(p, srv.url("/utf8-content-disposition/file/8K"));
VT_REQUIRE(r.has_value());
VT_CHECK_EQ(r.value().filename_from_disposition, kEuro + " rates.pdf");
VT_CHECK_EQ(suggest_filename(r.value()), kEuro + " rates.pdf");
}
VT_TEST(probe_reads_legacy_content_disposition) {
TestServer srv;
VT_REQUIRE(srv.available());
Prober p(4);
auto r = run_probe(p, srv.url("/legacy-content-disposition/file/8K"));
VT_REQUIRE(r.has_value());
VT_CHECK(!r.value().filename_from_disposition.empty()); // decoded, not the raw =?...?=
VT_CHECK(r.value().filename_from_disposition.find("=?") == std::string::npos);
}
VT_TEST(probe_follows_redirect_and_reports_effective_url) {
TestServer srv;
VT_REQUIRE(srv.available());
Prober p(4);
auto r = run_probe(p, srv.url("/redirect-chain/file/64K"));
VT_REQUIRE(r.has_value());
const auto &pr = r.value();
VT_CHECK(pr.effective_url != srv.url("/redirect-chain/file/64K"));
VT_REQUIRE(pr.redirect_chain.size() >= 2);
VT_CHECK_EQ(pr.redirect_chain.front(), srv.url("/redirect-chain/file/64K"));
VT_CHECK_EQ(pr.redirect_chain.back(), pr.effective_url);
}
VT_TEST(probe_401_is_requires_auth_not_error) {
TestServer srv;
VT_REQUIRE(srv.available());
Prober p(4);
auto r = run_probe(p, srv.url("/401-basic/file/64K"));
VT_REQUIRE(r.has_value()); // success result...
VT_CHECK(r.value().requires_auth); // ...flagged for the credential dialog
}
VT_TEST(probe_404_is_an_error) {
TestServer srv;
VT_REQUIRE(srv.available());
Prober p(4);
auto r = run_probe(p, srv.url("/plain/nope"));
VT_REQUIRE(!r.has_value());
VT_CHECK_EQ(r.error().code, Error::not_found);
}
VT_TEST(probe_dead_host_is_connect_failed) {
Prober p(4);
std::promise<Result<ProbeResult>> prom;
auto fut = prom.get_future();
ProbeRequest req;
req.url = "http://127.0.0.1:1/x";
req.connect_timeout_ms = 2000;
p.probe(std::move(req), [&](Result<ProbeResult> r) { prom.set_value(std::move(r)); });
VT_REQUIRE(fut.wait_for(std::chrono::seconds(10)) == std::future_status::ready);
auto r = fut.get();
VT_REQUIRE(!r.has_value());
VT_CHECK_EQ(r.error().code, Error::connect_failed);
}
VT_TEST(probe_pool_serialises_excess_requests) {
TestServer srv;
VT_REQUIRE(srv.available());
Prober p(2); // only 2 at a time
constexpr int kN = 8;
std::vector<std::future<Result<ProbeResult>>> futs;
std::vector<std::promise<Result<ProbeResult>>> proms(kN);
for (int i = 0; i < kN; ++i) {
futs.push_back(proms[i].get_future());
ProbeRequest req;
req.url = srv.url("/plain/file/4K");
p.probe(std::move(req),
[pr = &proms[i]](Result<ProbeResult> r) { pr->set_value(std::move(r)); });
}
for (auto &f : futs) {
VT_REQUIRE(f.wait_for(std::chrono::seconds(30)) == std::future_status::ready);
VT_CHECK(f.get().has_value());
}
}
core/tests/net/url_test.cpp
+71
View File
@@ -0,0 +1,71 @@
#include "vdm/net/url.hpp"
#include <string>
#include "vtest.hpp"
using vdm::net::split_url;
using vdm::net::SplitUrl;
using vdm::net::url_filename;
VT_TEST(url_basic_https) {
auto u = split_url("https://example.com/path/to/file.iso?x=1#frag");
VT_CHECK(u.valid);
VT_CHECK_EQ(u.scheme, std::string("https"));
VT_CHECK_EQ(u.host, std::string("example.com"));
VT_CHECK(!u.port.has_value());
VT_CHECK_EQ(u.path, std::string("/path/to/file.iso"));
VT_CHECK_EQ(u.query, std::string("x=1"));
VT_CHECK_EQ(u.fragment, std::string("frag"));
}
VT_TEST(url_port_userinfo_lowercasing) {
auto u = split_url("HTTP://User:[email protected]:8080/a");
VT_CHECK_EQ(u.scheme, std::string("http"));
VT_CHECK_EQ(u.userinfo, std::string("User:pw"));
VT_CHECK_EQ(u.host, std::string("host.example.com"));
VT_REQUIRE(u.port.has_value());
VT_CHECK_EQ(*u.port, 8080);
}
VT_TEST(url_ipv6_host) {
auto u = split_url("http://[2001:db8::1]:9000/file");
VT_CHECK_EQ(u.host, std::string("2001:db8::1"));
VT_REQUIRE(u.port.has_value());
VT_CHECK_EQ(*u.port, 9000);
}
VT_TEST(url_no_path) {
auto u = split_url("https://example.com");
VT_CHECK(u.valid);
VT_CHECK_EQ(u.path, std::string(""));
}
VT_TEST(url_non_http_is_invalid_but_parsed) {
auto u = split_url("ftp://host/file");
VT_CHECK(!u.valid); // not http/https
VT_CHECK_EQ(u.scheme, std::string("ftp"));
VT_CHECK(!u.is_http());
}
VT_TEST(url_garbage_does_not_crash) {
VT_CHECK(!split_url("").valid);
VT_CHECK(!split_url("not a url").valid);
VT_CHECK(!split_url("://noscheme/x").valid);
VT_CHECK(!split_url("http://").valid);
auto a = split_url("http://////");
auto b = split_url("https://h/%%%/%");
(void)a;
(void)b;
}
VT_TEST(url_filename_extraction) {
VT_CHECK_EQ(url_filename("https://x.com/a/b/report%20final.pdf"),
std::string("report final.pdf"));
VT_CHECK_EQ(url_filename("https://x.com/a/b/file.iso?sig=abc"), std::string("file.iso"));
VT_CHECK_EQ(url_filename("https://x.com/dir/"), std::string(""));
VT_CHECK_EQ(url_filename("https://x.com"), std::string(""));
VT_CHECK_EQ(url_filename("https://x.com/%2e%2e"), std::string("")); // ".." rejected
VT_CHECK_EQ(url_filename("https://x.com/a%2Fb"),
std::string("a_b")); // decoded '/' neutralised
}