core: net/probe + Content-Disposition parser + URL splitter (stage 3)

net/content_disposition — total parser for the mojibake-prone header: RFC 6266 filename (quoted/token), RFC 5987 filename* ext-values (charset'lang'pct-encoded, incl. RFC 2231 continuations), legacy RFC 2047 encoded-words (=?UTF-8?B?..?= / ?Q?), and raw Latin-1 bytes; prefers filename* over filename; strips path components AFTER decoding (a base64 payload can hold '/'). 22-case test table. net/text_codec (internal) — percent-decode, UTF-8 validation, Latin-1-> UTF-8, base64, RFC 2047 — shared by the CD parser and the URL splitter. net/url — a small total URL splitter (scheme/userinfo/host/port/path/ query/fragment, http(s) validity) and url_filename() for the last path segment; used for the filename fallback. net/probe — HEAD then a ranged GET bytes=0-0 that PROVES resumability (206 + matching Content-Range + a validator), rather than trusting Accept-Ranges which servers lie about; the ranged GET is also the HEAD- refused (403/405/501) fallback. 401/407 -> success result with requires_auth, not an error. Runs on its own pool (max_concurrent, default 4) outside the segment budget per ADR 0011 §5. suggest_filename() does the resolution order (explicit -> disposition -> URL -> download.bin) with a light strip; rules/ (stage 9) owns the authoritative sanitize. tools/fuzz — libFuzzer targets for the CD parser and the URL splitter, compiling the parser sources directly so they're fully instrumented; self-guards on VELOX_BUILD_FUZZ + Clang (the top-level CMake adds every tools/* unconditionally). Seed corpora included. Fixed on the way: a p -> Transfer -> State -> cbs -> p reference cycle in Prober that leaked every probe (drop the stored Transfer; the worker keeps State alive). Tests green under ASan/UBSan and TSan. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01HPPSGhiArbvQgwC2DNiURS
2026-09-09 23:53:55 +04:00
parent fdacf732fa
commit 201ebc55d4
25 changed files with 1731 additions and 19 deletions
@@ -23,14 +23,21 @@ vdm_add_test(veloxcore_log_test       util/log_test.cpp)

 # net/ integration tests drive tools/testserver (lane PKG/QA). Skip cleanly if it isn't
 # in the tree yet (lanes merge independently).
+vdm_add_test(veloxcore_content_disposition_test net/content_disposition_test.cpp)
+vdm_add_test(veloxcore_url_test net/url_test.cpp)
+
 set(_testserver ${CMAKE_SOURCE_DIR}/tools/testserver/testserver.py)
-vdm_add_test(veloxcore_http_client_test net/http_client_test.cpp)
-target_include_directories(veloxcore_http_client_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/net)
-if(EXISTS ${_testserver})
-    target_compile_definitions(veloxcore_http_client_test
-        PRIVATE VDM_TESTSERVER_PY="${_testserver}")
-    set_tests_properties(veloxcore_http_client_test PROPERTIES TIMEOUT 120)
-else()
-    message(STATUS "veloxcore: tools/testserver not present; http_client_test will skip "
-                   "its server-backed cases.")
+foreach(net_it http_client probe)
+    vdm_add_test(veloxcore_${net_it}_test net/${net_it}_test.cpp)
+    target_include_directories(veloxcore_${net_it}_test
+        PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/net)
+    if(EXISTS ${_testserver})
+        target_compile_definitions(veloxcore_${net_it}_test
+            PRIVATE VDM_TESTSERVER_PY="${_testserver}")
+        set_tests_properties(veloxcore_${net_it}_test PROPERTIES TIMEOUT 120)
+    endif()
+endforeach()
+if(NOT EXISTS ${_testserver})
+    message(STATUS "veloxcore: tools/testserver not present; net integration tests will "
+                   "skip their server-backed cases.")
 endif()
@@ -0,0 +1,148 @@
+#include "vdm/net/content_disposition.hpp"
+
+#include <string>
+#include <string_view>
+
+#include "vtest.hpp"
+
+using vdm::net::ContentDisposition;
+using vdm::net::parse_content_disposition;
+using Type = vdm::net::ContentDisposition::Type;
+
+namespace {
+// "€" is U+20AC -> UTF-8 E2 82 AC. "£" is U+00A3 -> UTF-8 C2 A3.
+const std::string kEuro = "\xE2\x82\xAC";
+const std::string kPound = "\xC2\xA3";
+const std::string kEAcute = "\xC3\xA9";  // é U+00E9
+}  // namespace
+
+VT_TEST(cd_plain_quoted) {
+    auto cd = parse_content_disposition(R"(attachment; filename="report.pdf")");
+    VT_CHECK(cd.type == Type::attachment);
+    VT_CHECK_EQ(cd.filename, std::string("report.pdf"));
+    VT_CHECK(!cd.filename_from_ext);
+}
+
+VT_TEST(cd_plain_token_unquoted) {
+    auto cd = parse_content_disposition("attachment; filename=report.pdf");
+    VT_CHECK_EQ(cd.filename, std::string("report.pdf"));
+}
+
+VT_TEST(cd_inline_no_filename) {
+    auto cd = parse_content_disposition("inline");
+    VT_CHECK(cd.type == Type::inline_);
+    VT_CHECK(!cd.has_filename());
+}
+
+VT_TEST(cd_rfc5987_utf8_ext_value) {
+    auto cd = parse_content_disposition("attachment; filename*=UTF-8''%e2%82%ac%20rates.pdf");
+    VT_CHECK_EQ(cd.filename, kEuro + " rates.pdf");
+    VT_CHECK(cd.filename_from_ext);
+}
+
+VT_TEST(cd_prefers_ext_over_plain) {
+    auto cd = parse_content_disposition(
+        R"(attachment; filename="EURO rates.pdf"; filename*=UTF-8''%e2%82%ac%20rates.pdf)");
+    VT_CHECK_EQ(cd.filename, kEuro + " rates.pdf");
+    VT_CHECK(cd.filename_from_ext);
+}
+
+VT_TEST(cd_rfc5987_latin1_ext_value) {
+    // %A3 = £ in ISO-8859-1
+    auto cd = parse_content_disposition("attachment; filename*=ISO-8859-1''%A3rates.pdf");
+    VT_CHECK_EQ(cd.filename, kPound + "rates.pdf");
+}
+
+VT_TEST(cd_legacy_rfc2047_base64) {
+    // base64("<euro> rates.pdf") with euro as UTF-8
+    // "€ rates.pdf" -> bytes E2 82 AC 20 72 61 74 65 73 2E 70 64 66 -> base64:
+    auto cd =
+        parse_content_disposition(R"(attachment; filename="=?UTF-8?B?4oKsIHJhdGVzLnBkZg==?=")");
+    VT_CHECK_EQ(cd.filename, kEuro + " rates.pdf");
+}
+
+VT_TEST(cd_legacy_rfc2047_qencoded_latin1) {
+    // =?ISO-8859-1?Q?=A3rates.pdf?=  -> £rates.pdf
+    auto cd = parse_content_disposition(R"(attachment; filename="=?ISO-8859-1?Q?=A3rates.pdf?=")");
+    VT_CHECK_EQ(cd.filename, kPound + "rates.pdf");
+}
+
+VT_TEST(cd_raw_utf8_bytes_in_quotes) {
+    std::string h = "attachment; filename=\"caf" + kEAcute + ".txt\"";
+    auto cd = parse_content_disposition(h);
+    VT_CHECK_EQ(cd.filename, "caf" + kEAcute + ".txt");
+}
+
+VT_TEST(cd_raw_latin1_byte_in_quotes) {
+    // 0xE9 is 'é' in Latin-1; not valid UTF-8 alone -> transcoded
+    std::string h = "attachment; filename=\"caf\xE9.txt\"";
+    auto cd = parse_content_disposition(h);
+    VT_CHECK_EQ(cd.filename, "caf" + kEAcute + ".txt");
+}
+
+VT_TEST(cd_strips_path_components) {
+    VT_CHECK_EQ(parse_content_disposition(R"(attachment; filename="../../etc/passwd")").filename,
+                std::string("passwd"));
+    // real Windows paths in the wild use unescaped backslashes as separators
+    VT_CHECK_EQ(parse_content_disposition(R"(attachment; filename="C:\Windows\evil.exe")").filename,
+                std::string("evil.exe"));
+    // a base64 payload can contain '/', so decode must happen before path stripping
+    VT_CHECK_EQ(parse_content_disposition(R"(attachment; filename="=?UTF-8?B?Li4vLi4vc2VjcmV0?=")")
+                    .filename,
+                std::string("secret"));  // decodes to "../../secret", then stripped
+}
+
+VT_TEST(cd_quoted_dquote_escape) {
+    // \" is the one escape we resolve, so a quote can appear mid-name
+    auto cd = parse_content_disposition(R"(attachment; filename="quote\"here.txt")");
+    VT_CHECK_EQ(cd.filename, std::string("quote\"here.txt"));
+}
+
+VT_TEST(cd_semicolon_inside_quotes_is_not_a_separator) {
+    auto cd = parse_content_disposition(R"(attachment; filename="a;b;c.txt")");
+    VT_CHECK_EQ(cd.filename, std::string("a;b;c.txt"));
+}
+
+VT_TEST(cd_form_data) {
+    auto cd = parse_content_disposition(R"(form-data; name="file"; filename="upload.bin")");
+    VT_CHECK(cd.type == Type::form_data);
+    VT_CHECK_EQ(cd.filename, std::string("upload.bin"));
+}
+
+VT_TEST(cd_rfc2231_continuations) {
+    auto cd = parse_content_disposition(
+        "attachment; filename*0*=UTF-8''%e2%82%ac; filename*1*=%20rates; filename*2=.pdf");
+    VT_CHECK_EQ(cd.filename, kEuro + " rates.pdf");
+}
+
+VT_TEST(cd_empty_and_garbage_do_not_crash) {
+    VT_CHECK(!parse_content_disposition("").has_filename());
+    VT_CHECK(!parse_content_disposition(";;;;").has_filename());
+    VT_CHECK(!parse_content_disposition("attachment;").has_filename());
+    VT_CHECK(!parse_content_disposition(R"(attachment; filename=)").has_filename());
+    VT_CHECK(!parse_content_disposition(R"(attachment; filename="")").has_filename());
+    // truncated ext-value
+    auto cd = parse_content_disposition("attachment; filename*=UTF-8''%e2%82");
+    VT_CHECK(cd.type == Type::attachment);  // no crash; filename is whatever fell out
+    // truncated encoded-word
+    auto trunc = parse_content_disposition(R"(attachment; filename="=?UTF-8?B?4oKs")");
+    (void)trunc;
+}
+
+VT_TEST(cd_bad_percent_escapes_in_ext_value) {
+    // stray % and non-hex digits are emitted literally, no crash
+    auto cd = parse_content_disposition("attachment; filename*=UTF-8''%ZZ%%file%2");
+    VT_CHECK(cd.type == Type::attachment);
+}
+
+VT_TEST(cd_case_insensitive_keys_and_type) {
+    auto cd = parse_content_disposition(R"(ATTACHMENT; FileName="x.txt")");
+    VT_CHECK(cd.type == Type::attachment);
+    VT_CHECK_EQ(cd.filename, std::string("x.txt"));
+}
+
+VT_TEST(cd_unknown_type_is_other) {
+    auto cd = parse_content_disposition(R"(signal; filename="x.txt")");
+    VT_CHECK(cd.type == Type::other);
+    VT_CHECK_EQ(cd.filename, std::string("x.txt"));
+}
@@ -0,0 +1,163 @@
+#include "vdm/net/probe.hpp"
+
+#include <chrono>
+#include <future>
+#include <string>
+
+#include "testserver_fixture.hpp"
+#include "vtest.hpp"
+
+using namespace vdm;
+using namespace vdm::net;
+using vdm::testing::TestServer;
+
+namespace {
+Result<ProbeResult> run_probe(Prober &p, const std::string &url) {
+    std::promise<Result<ProbeResult>> prom;
+    auto fut = prom.get_future();
+    ProbeRequest req;
+    req.url = url;
+    req.overall_timeout_ms = 15000;
+    p.probe(std::move(req), [&](Result<ProbeResult> r) { prom.set_value(std::move(r)); });
+    if (fut.wait_for(std::chrono::seconds(25)) != std::future_status::ready)
+        return Err{Error::timeout, "probe test wait"};
+    return fut.get();
+}
+const std::string kEuro = "\xE2\x82\xAC";
+}  // namespace
+
+VT_TEST(probe_plain_resumable_file) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/plain/file/2M"));
+    VT_REQUIRE(r.has_value());
+    const auto &pr = r.value();
+    VT_CHECK_EQ(pr.http_status, 206);  // proven via the ranged GET
+    VT_CHECK(pr.accept_ranges);
+    VT_CHECK(pr.resumable);  // 206 + ETag validator
+    VT_REQUIRE(pr.total_size.has_value());
+    VT_CHECK_EQ(*pr.total_size, 2u * 1024 * 1024);
+    VT_CHECK(!pr.etag.empty());
+    VT_CHECK_EQ(pr.filename_from_url, std::string("2M"));
+    VT_CHECK(!pr.requires_auth);
+}
+
+VT_TEST(probe_no_range_server_is_not_resumable) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/no-range/file/1M"));
+    VT_REQUIRE(r.has_value());
+    const auto &pr = r.value();
+    VT_CHECK(!pr.resumable);  // no 206 ever
+    VT_CHECK(!pr.accept_ranges);
+    VT_REQUIRE(pr.total_size.has_value());
+    VT_CHECK_EQ(*pr.total_size, 1u * 1024 * 1024);
+}
+
+VT_TEST(probe_lying_accept_ranges_still_not_resumable) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    // advertises Accept-Ranges: bytes but never returns 206
+    auto r = run_probe(p, srv.url("/lies-about-accept-ranges/file/1M"));
+    VT_REQUIRE(r.has_value());
+    const auto &pr = r.value();
+    VT_CHECK(pr.accept_ranges);  // it advertised
+    VT_CHECK(!pr.resumable);     // ...but never proved it -> resume is off
+}
+
+VT_TEST(probe_reads_utf8_content_disposition) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/utf8-content-disposition/file/8K"));
+    VT_REQUIRE(r.has_value());
+    VT_CHECK_EQ(r.value().filename_from_disposition, kEuro + " rates.pdf");
+    VT_CHECK_EQ(suggest_filename(r.value()), kEuro + " rates.pdf");
+}
+
+VT_TEST(probe_reads_legacy_content_disposition) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/legacy-content-disposition/file/8K"));
+    VT_REQUIRE(r.has_value());
+    VT_CHECK(!r.value().filename_from_disposition.empty());  // decoded, not the raw =?...?=
+    VT_CHECK(r.value().filename_from_disposition.find("=?") == std::string::npos);
+}
+
+VT_TEST(probe_follows_redirect_and_reports_effective_url) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/redirect-chain/file/64K"));
+    VT_REQUIRE(r.has_value());
+    const auto &pr = r.value();
+    VT_CHECK(pr.effective_url != srv.url("/redirect-chain/file/64K"));
+    VT_REQUIRE(pr.redirect_chain.size() >= 2);
+    VT_CHECK_EQ(pr.redirect_chain.front(), srv.url("/redirect-chain/file/64K"));
+    VT_CHECK_EQ(pr.redirect_chain.back(), pr.effective_url);
+}
+
+VT_TEST(probe_401_is_requires_auth_not_error) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/401-basic/file/64K"));
+    VT_REQUIRE(r.has_value());          // success result...
+    VT_CHECK(r.value().requires_auth);  // ...flagged for the credential dialog
+}
+
+VT_TEST(probe_404_is_an_error) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/plain/nope"));
+    VT_REQUIRE(!r.has_value());
+    VT_CHECK_EQ(r.error().code, Error::not_found);
+}
+
+VT_TEST(probe_dead_host_is_connect_failed) {
+    Prober p(4);
+    std::promise<Result<ProbeResult>> prom;
+    auto fut = prom.get_future();
+    ProbeRequest req;
+    req.url = "http://127.0.0.1:1/x";
+    req.connect_timeout_ms = 2000;
+    p.probe(std::move(req), [&](Result<ProbeResult> r) { prom.set_value(std::move(r)); });
+    VT_REQUIRE(fut.wait_for(std::chrono::seconds(10)) == std::future_status::ready);
+    auto r = fut.get();
+    VT_REQUIRE(!r.has_value());
+    VT_CHECK_EQ(r.error().code, Error::connect_failed);
+}
+
+VT_TEST(probe_pool_serialises_excess_requests) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(2);  // only 2 at a time
+
+    constexpr int kN = 8;
+    std::vector<std::future<Result<ProbeResult>>> futs;
+    std::vector<std::promise<Result<ProbeResult>>> proms(kN);
+    for (int i = 0; i < kN; ++i) {
+        futs.push_back(proms[i].get_future());
+        ProbeRequest req;
+        req.url = srv.url("/plain/file/4K");
+        p.probe(std::move(req),
+                [pr = &proms[i]](Result<ProbeResult> r) { pr->set_value(std::move(r)); });
+    }
+    for (auto &f : futs) {
+        VT_REQUIRE(f.wait_for(std::chrono::seconds(30)) == std::future_status::ready);
+        VT_CHECK(f.get().has_value());
+    }
+}
@@ -0,0 +1,71 @@
+#include "vdm/net/url.hpp"
+
+#include <string>
+
+#include "vtest.hpp"
+
+using vdm::net::split_url;
+using vdm::net::SplitUrl;
+using vdm::net::url_filename;
+
+VT_TEST(url_basic_https) {
+    auto u = split_url("https://example.com/path/to/file.iso?x=1#frag");
+    VT_CHECK(u.valid);
+    VT_CHECK_EQ(u.scheme, std::string("https"));
+    VT_CHECK_EQ(u.host, std::string("example.com"));
+    VT_CHECK(!u.port.has_value());
+    VT_CHECK_EQ(u.path, std::string("/path/to/file.iso"));
+    VT_CHECK_EQ(u.query, std::string("x=1"));
+    VT_CHECK_EQ(u.fragment, std::string("frag"));
+}
+
+VT_TEST(url_port_userinfo_lowercasing) {
+    auto u = split_url("HTTP://User:[email protected]:8080/a");
+    VT_CHECK_EQ(u.scheme, std::string("http"));
+    VT_CHECK_EQ(u.userinfo, std::string("User:pw"));
+    VT_CHECK_EQ(u.host, std::string("host.example.com"));
+    VT_REQUIRE(u.port.has_value());
+    VT_CHECK_EQ(*u.port, 8080);
+}
+
+VT_TEST(url_ipv6_host) {
+    auto u = split_url("http://[2001:db8::1]:9000/file");
+    VT_CHECK_EQ(u.host, std::string("2001:db8::1"));
+    VT_REQUIRE(u.port.has_value());
+    VT_CHECK_EQ(*u.port, 9000);
+}
+
+VT_TEST(url_no_path) {
+    auto u = split_url("https://example.com");
+    VT_CHECK(u.valid);
+    VT_CHECK_EQ(u.path, std::string(""));
+}
+
+VT_TEST(url_non_http_is_invalid_but_parsed) {
+    auto u = split_url("ftp://host/file");
+    VT_CHECK(!u.valid);  // not http/https
+    VT_CHECK_EQ(u.scheme, std::string("ftp"));
+    VT_CHECK(!u.is_http());
+}
+
+VT_TEST(url_garbage_does_not_crash) {
+    VT_CHECK(!split_url("").valid);
+    VT_CHECK(!split_url("not a url").valid);
+    VT_CHECK(!split_url("://noscheme/x").valid);
+    VT_CHECK(!split_url("http://").valid);
+    auto a = split_url("http://////");
+    auto b = split_url("https://h/%%%/%");
+    (void)a;
+    (void)b;
+}
+
+VT_TEST(url_filename_extraction) {
+    VT_CHECK_EQ(url_filename("https://x.com/a/b/report%20final.pdf"),
+                std::string("report final.pdf"));
+    VT_CHECK_EQ(url_filename("https://x.com/a/b/file.iso?sig=abc"), std::string("file.iso"));
+    VT_CHECK_EQ(url_filename("https://x.com/dir/"), std::string(""));
+    VT_CHECK_EQ(url_filename("https://x.com"), std::string(""));
+    VT_CHECK_EQ(url_filename("https://x.com/%2e%2e"), std::string(""));  // ".." rejected
+    VT_CHECK_EQ(url_filename("https://x.com/a%2Fb"),
+                std::string("a_b"));  // decoded '/' neutralised
+}