core: net/probe + Content-Disposition parser + URL splitter (stage 3)

net/content_disposition — total parser for the mojibake-prone header: RFC 6266 filename (quoted/token), RFC 5987 filename* ext-values (charset'lang'pct-encoded, incl. RFC 2231 continuations), legacy RFC 2047 encoded-words (=?UTF-8?B?..?= / ?Q?), and raw Latin-1 bytes; prefers filename* over filename; strips path components AFTER decoding (a base64 payload can hold '/'). 22-case test table. net/text_codec (internal) — percent-decode, UTF-8 validation, Latin-1-> UTF-8, base64, RFC 2047 — shared by the CD parser and the URL splitter. net/url — a small total URL splitter (scheme/userinfo/host/port/path/ query/fragment, http(s) validity) and url_filename() for the last path segment; used for the filename fallback. net/probe — HEAD then a ranged GET bytes=0-0 that PROVES resumability (206 + matching Content-Range + a validator), rather than trusting Accept-Ranges which servers lie about; the ranged GET is also the HEAD- refused (403/405/501) fallback. 401/407 -> success result with requires_auth, not an error. Runs on its own pool (max_concurrent, default 4) outside the segment budget per ADR 0011 §5. suggest_filename() does the resolution order (explicit -> disposition -> URL -> download.bin) with a light strip; rules/ (stage 9) owns the authoritative sanitize. tools/fuzz — libFuzzer targets for the CD parser and the URL splitter, compiling the parser sources directly so they're fully instrumented; self-guards on VELOX_BUILD_FUZZ + Clang (the top-level CMake adds every tools/* unconditionally). Seed corpora included. Fixed on the way: a p -> Transfer -> State -> cbs -> p reference cycle in Prober that leaked every probe (drop the stored Transfer; the worker keeps State alive). Tests green under ASan/UBSan and TSan. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01HPPSGhiArbvQgwC2DNiURS
2026-09-09 23:53:55 +04:00
parent fdacf732fa
commit 201ebc55d4
25 changed files with 1731 additions and 19 deletions
@@ -0,0 +1,163 @@
+#include "vdm/net/probe.hpp"
+
+#include <chrono>
+#include <future>
+#include <string>
+
+#include "testserver_fixture.hpp"
+#include "vtest.hpp"
+
+using namespace vdm;
+using namespace vdm::net;
+using vdm::testing::TestServer;
+
+namespace {
+Result<ProbeResult> run_probe(Prober &p, const std::string &url) {
+    std::promise<Result<ProbeResult>> prom;
+    auto fut = prom.get_future();
+    ProbeRequest req;
+    req.url = url;
+    req.overall_timeout_ms = 15000;
+    p.probe(std::move(req), [&](Result<ProbeResult> r) { prom.set_value(std::move(r)); });
+    if (fut.wait_for(std::chrono::seconds(25)) != std::future_status::ready)
+        return Err{Error::timeout, "probe test wait"};
+    return fut.get();
+}
+const std::string kEuro = "\xE2\x82\xAC";
+}  // namespace
+
+VT_TEST(probe_plain_resumable_file) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/plain/file/2M"));
+    VT_REQUIRE(r.has_value());
+    const auto &pr = r.value();
+    VT_CHECK_EQ(pr.http_status, 206);  // proven via the ranged GET
+    VT_CHECK(pr.accept_ranges);
+    VT_CHECK(pr.resumable);  // 206 + ETag validator
+    VT_REQUIRE(pr.total_size.has_value());
+    VT_CHECK_EQ(*pr.total_size, 2u * 1024 * 1024);
+    VT_CHECK(!pr.etag.empty());
+    VT_CHECK_EQ(pr.filename_from_url, std::string("2M"));
+    VT_CHECK(!pr.requires_auth);
+}
+
+VT_TEST(probe_no_range_server_is_not_resumable) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/no-range/file/1M"));
+    VT_REQUIRE(r.has_value());
+    const auto &pr = r.value();
+    VT_CHECK(!pr.resumable);  // no 206 ever
+    VT_CHECK(!pr.accept_ranges);
+    VT_REQUIRE(pr.total_size.has_value());
+    VT_CHECK_EQ(*pr.total_size, 1u * 1024 * 1024);
+}
+
+VT_TEST(probe_lying_accept_ranges_still_not_resumable) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    // advertises Accept-Ranges: bytes but never returns 206
+    auto r = run_probe(p, srv.url("/lies-about-accept-ranges/file/1M"));
+    VT_REQUIRE(r.has_value());
+    const auto &pr = r.value();
+    VT_CHECK(pr.accept_ranges);  // it advertised
+    VT_CHECK(!pr.resumable);     // ...but never proved it -> resume is off
+}
+
+VT_TEST(probe_reads_utf8_content_disposition) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/utf8-content-disposition/file/8K"));
+    VT_REQUIRE(r.has_value());
+    VT_CHECK_EQ(r.value().filename_from_disposition, kEuro + " rates.pdf");
+    VT_CHECK_EQ(suggest_filename(r.value()), kEuro + " rates.pdf");
+}
+
+VT_TEST(probe_reads_legacy_content_disposition) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/legacy-content-disposition/file/8K"));
+    VT_REQUIRE(r.has_value());
+    VT_CHECK(!r.value().filename_from_disposition.empty());  // decoded, not the raw =?...?=
+    VT_CHECK(r.value().filename_from_disposition.find("=?") == std::string::npos);
+}
+
+VT_TEST(probe_follows_redirect_and_reports_effective_url) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/redirect-chain/file/64K"));
+    VT_REQUIRE(r.has_value());
+    const auto &pr = r.value();
+    VT_CHECK(pr.effective_url != srv.url("/redirect-chain/file/64K"));
+    VT_REQUIRE(pr.redirect_chain.size() >= 2);
+    VT_CHECK_EQ(pr.redirect_chain.front(), srv.url("/redirect-chain/file/64K"));
+    VT_CHECK_EQ(pr.redirect_chain.back(), pr.effective_url);
+}
+
+VT_TEST(probe_401_is_requires_auth_not_error) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/401-basic/file/64K"));
+    VT_REQUIRE(r.has_value());          // success result...
+    VT_CHECK(r.value().requires_auth);  // ...flagged for the credential dialog
+}
+
+VT_TEST(probe_404_is_an_error) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(4);
+
+    auto r = run_probe(p, srv.url("/plain/nope"));
+    VT_REQUIRE(!r.has_value());
+    VT_CHECK_EQ(r.error().code, Error::not_found);
+}
+
+VT_TEST(probe_dead_host_is_connect_failed) {
+    Prober p(4);
+    std::promise<Result<ProbeResult>> prom;
+    auto fut = prom.get_future();
+    ProbeRequest req;
+    req.url = "http://127.0.0.1:1/x";
+    req.connect_timeout_ms = 2000;
+    p.probe(std::move(req), [&](Result<ProbeResult> r) { prom.set_value(std::move(r)); });
+    VT_REQUIRE(fut.wait_for(std::chrono::seconds(10)) == std::future_status::ready);
+    auto r = fut.get();
+    VT_REQUIRE(!r.has_value());
+    VT_CHECK_EQ(r.error().code, Error::connect_failed);
+}
+
+VT_TEST(probe_pool_serialises_excess_requests) {
+    TestServer srv;
+    VT_REQUIRE(srv.available());
+    Prober p(2);  // only 2 at a time
+
+    constexpr int kN = 8;
+    std::vector<std::future<Result<ProbeResult>>> futs;
+    std::vector<std::promise<Result<ProbeResult>>> proms(kN);
+    for (int i = 0; i < kN; ++i) {
+        futs.push_back(proms[i].get_future());
+        ProbeRequest req;
+        req.url = srv.url("/plain/file/4K");
+        p.probe(std::move(req),
+                [pr = &proms[i]](Result<ProbeResult> r) { pr->set_value(std::move(r)); });
+    }
+    for (auto &f : futs) {
+        VT_REQUIRE(f.wait_for(std::chrono::seconds(30)) == std::future_status::ready);
+        VT_CHECK(f.get().has_value());
+    }
+}