core: net/probe + Content-Disposition parser + URL splitter (stage 3)
net/content_disposition — total parser for the mojibake-prone header: RFC 6266 filename (quoted/token), RFC 5987 filename* ext-values (charset'lang'pct-encoded, incl. RFC 2231 continuations), legacy RFC 2047 encoded-words (=?UTF-8?B?..?= / ?Q?), and raw Latin-1 bytes; prefers filename* over filename; strips path components AFTER decoding (a base64 payload can hold '/'). 22-case test table. net/text_codec (internal) — percent-decode, UTF-8 validation, Latin-1-> UTF-8, base64, RFC 2047 — shared by the CD parser and the URL splitter. net/url — a small total URL splitter (scheme/userinfo/host/port/path/ query/fragment, http(s) validity) and url_filename() for the last path segment; used for the filename fallback. net/probe — HEAD then a ranged GET bytes=0-0 that PROVES resumability (206 + matching Content-Range + a validator), rather than trusting Accept-Ranges which servers lie about; the ranged GET is also the HEAD- refused (403/405/501) fallback. 401/407 -> success result with requires_auth, not an error. Runs on its own pool (max_concurrent, default 4) outside the segment budget per ADR 0011 §5. suggest_filename() does the resolution order (explicit -> disposition -> URL -> download.bin) with a light strip; rules/ (stage 9) owns the authoritative sanitize. tools/fuzz — libFuzzer targets for the CD parser and the URL splitter, compiling the parser sources directly so they're fully instrumented; self-guards on VELOX_BUILD_FUZZ + Clang (the top-level CMake adds every tools/* unconditionally). Seed corpora included. Fixed on the way: a p -> Transfer -> State -> cbs -> p reference cycle in Prober that leaked every probe (drop the stored Transfer; the worker keeps State alive). Tests green under ASan/UBSan and TSan. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01HPPSGhiArbvQgwC2DNiURS
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,348 @@
|
||||
// vdm/net/probe.cpp
|
||||
|
||||
#include "vdm/net/probe.hpp"
|
||||
|
||||
#include <charconv>
|
||||
#include <deque>
|
||||
#include <mutex>
|
||||
|
||||
#include "net/curl_error.hpp"
|
||||
#include "vdm/net/http_client.hpp"
|
||||
#include "vdm/net/url.hpp"
|
||||
|
||||
namespace vdm::net {
|
||||
namespace {
|
||||
|
||||
std::string_view trim(std::string_view s) {
|
||||
while (!s.empty() && (s.front() == ' ' || s.front() == '\t'))
|
||||
s.remove_prefix(1);
|
||||
while (!s.empty() && (s.back() == ' ' || s.back() == '\t'))
|
||||
s.remove_suffix(1);
|
||||
return s;
|
||||
}
|
||||
|
||||
bool iequals(std::string_view a, std::string_view b) {
|
||||
return HeaderList::iequals(a, b);
|
||||
}
|
||||
|
||||
// "bytes 0-0/12345" -> 12345 ; "bytes 0-0/*" or malformed -> nullopt
|
||||
std::optional<std::uint64_t> total_from_content_range(std::string_view v) {
|
||||
auto slash = v.find('/');
|
||||
if (slash == std::string_view::npos)
|
||||
return std::nullopt;
|
||||
std::string_view tail = trim(v.substr(slash + 1));
|
||||
if (tail.empty() || tail == "*")
|
||||
return std::nullopt;
|
||||
std::uint64_t n = 0;
|
||||
auto [p, ec] = std::from_chars(tail.data(), tail.data() + tail.size(), n);
|
||||
(void)p;
|
||||
if (ec != std::errc{})
|
||||
return std::nullopt;
|
||||
return n;
|
||||
}
|
||||
|
||||
std::string light_sanitize(std::string_view in) {
|
||||
std::string out;
|
||||
out.reserve(in.size());
|
||||
for (unsigned char c : in) {
|
||||
if (c == '/' || c == '\\' || c == 0) {
|
||||
out.push_back('_');
|
||||
} else if (c >= 0x20 || (c & 0x80)) { // keep printable ASCII + all UTF-8 bytes
|
||||
out.push_back(static_cast<char>(c));
|
||||
}
|
||||
}
|
||||
while (!out.empty() && (out.back() == '.' || out.back() == ' '))
|
||||
out.pop_back();
|
||||
if (out == "." || out == "..")
|
||||
out.clear();
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string suggest_filename(const ProbeResult &r, std::string_view explicit_name) {
|
||||
std::string cand;
|
||||
if (!explicit_name.empty())
|
||||
cand = light_sanitize(explicit_name);
|
||||
if (cand.empty() && !r.filename_from_disposition.empty())
|
||||
cand = light_sanitize(r.filename_from_disposition);
|
||||
if (cand.empty() && !r.filename_from_url.empty())
|
||||
cand = light_sanitize(r.filename_from_url);
|
||||
if (cand.empty())
|
||||
cand = "download.bin";
|
||||
return cand;
|
||||
}
|
||||
|
||||
// --- Prober::Impl ---------------------------------------------------------------------
|
||||
struct Prober::Impl {
|
||||
struct Job {
|
||||
ProbeRequest req;
|
||||
std::function<void(Result<ProbeResult>)> done;
|
||||
};
|
||||
|
||||
struct P {
|
||||
Impl *self = nullptr;
|
||||
Job job;
|
||||
ProbeResult result;
|
||||
bool had_head_ok = false;
|
||||
bool delivered = false;
|
||||
};
|
||||
|
||||
explicit Impl(unsigned max_concurrent)
|
||||
: max_(max_concurrent ? max_concurrent : 1), client_(HttpClient::Options{.workers = 2}) {}
|
||||
|
||||
unsigned max_;
|
||||
HttpClient client_;
|
||||
std::mutex mu_;
|
||||
unsigned inflight_ = 0;
|
||||
std::deque<Job> pending_;
|
||||
|
||||
void submit(Job j) {
|
||||
{
|
||||
std::lock_guard lk(mu_);
|
||||
if (inflight_ >= max_) {
|
||||
pending_.push_back(std::move(j));
|
||||
return;
|
||||
}
|
||||
++inflight_;
|
||||
}
|
||||
start(std::move(j));
|
||||
}
|
||||
|
||||
void finish_one() {
|
||||
Job next;
|
||||
bool have_next = false;
|
||||
{
|
||||
std::lock_guard lk(mu_);
|
||||
--inflight_;
|
||||
if (!pending_.empty()) {
|
||||
next = std::move(pending_.front());
|
||||
pending_.pop_front();
|
||||
++inflight_;
|
||||
have_next = true;
|
||||
}
|
||||
}
|
||||
if (have_next)
|
||||
start(std::move(next));
|
||||
}
|
||||
|
||||
Request base_request(const ProbeRequest &pr) {
|
||||
Request r;
|
||||
r.url = pr.url;
|
||||
r.headers = pr.headers;
|
||||
r.cookies = pr.cookies;
|
||||
r.user_agent = pr.user_agent;
|
||||
r.referrer = pr.referrer;
|
||||
r.proxy = pr.proxy;
|
||||
r.follow_redirects = true;
|
||||
r.accept_encoding = false;
|
||||
r.connect_timeout_ms = pr.connect_timeout_ms;
|
||||
r.overall_timeout_ms = pr.overall_timeout_ms;
|
||||
r.low_speed_bytes_per_sec = 0; // probes are tiny; no stall detector
|
||||
r.low_speed_secs = 0;
|
||||
return r;
|
||||
}
|
||||
|
||||
void start(Job j) {
|
||||
auto p = std::make_shared<P>();
|
||||
p->self = this;
|
||||
p->job = std::move(j);
|
||||
p->result.effective_url = p->job.req.url;
|
||||
|
||||
Request req = base_request(p->job.req);
|
||||
req.method = Method::head;
|
||||
|
||||
TransferCallbacks cbs;
|
||||
cbs.on_head = [p](const ResponseHead &h) {
|
||||
absorb_head(p->result, h);
|
||||
return DataAction::abort;
|
||||
};
|
||||
cbs.on_data = [](ConstByteSpan) { return DataAction::abort; };
|
||||
cbs.on_finished = [p](Result<TransferStats> r) { on_head_done(p, std::move(r)); };
|
||||
|
||||
// The worker keeps Transfer::State alive; the callbacks keep `p` alive. Storing
|
||||
// the Transfer in `p` would make a p -> Transfer -> State -> cbs -> p cycle.
|
||||
client_.start(std::move(req), std::move(cbs));
|
||||
}
|
||||
|
||||
static void absorb_head(ProbeResult &res, const ResponseHead &h) {
|
||||
if (h.status)
|
||||
res.http_status = h.status;
|
||||
if (!h.effective_url.empty())
|
||||
res.effective_url = h.effective_url;
|
||||
refresh_common(res, h);
|
||||
if (h.content_length && !res.total_size)
|
||||
res.total_size = h.content_length;
|
||||
}
|
||||
|
||||
static void refresh_common(ProbeResult &res, const ResponseHead &h) {
|
||||
if (auto v = h.headers.get("Content-Type"); v && res.mime.empty()) {
|
||||
std::string_view mv = *v;
|
||||
mv = mv.substr(0, mv.find(';'));
|
||||
res.mime.assign(trim(mv));
|
||||
}
|
||||
if (auto v = h.headers.get("ETag"); v && res.etag.empty())
|
||||
res.etag.assign(*v);
|
||||
if (auto v = h.headers.get("Last-Modified"); v && res.last_modified.empty())
|
||||
res.last_modified.assign(*v);
|
||||
if (auto v = h.headers.get("Accept-Ranges")) {
|
||||
if (iequals(trim(*v), "bytes"))
|
||||
res.accept_ranges = true;
|
||||
}
|
||||
if (auto v = h.headers.get("Content-Disposition");
|
||||
v && res.filename_from_disposition.empty()) {
|
||||
ContentDisposition cd = parse_content_disposition(*v);
|
||||
res.disposition_type = cd.type;
|
||||
if (cd.has_filename())
|
||||
res.filename_from_disposition = cd.filename;
|
||||
}
|
||||
}
|
||||
|
||||
static bool has_validator(const ProbeResult &r) {
|
||||
return !r.etag.empty() || !r.last_modified.empty();
|
||||
}
|
||||
|
||||
static void on_head_done(std::shared_ptr<P> p, Result<TransferStats> r) {
|
||||
const long status = p->result.http_status;
|
||||
|
||||
if (status == 0) { // never got headers -> transport failure
|
||||
deliver(p, r.has_value() ? Result<ProbeResult>(ErrorInfo(Error::probe_failed))
|
||||
: Result<ProbeResult>(std::move(r).error()));
|
||||
return;
|
||||
}
|
||||
p->had_head_ok = (status >= 200 && status < 300);
|
||||
|
||||
if (status == 401 || status == 407) {
|
||||
p->result.requires_auth = true;
|
||||
finalize_and_deliver(p);
|
||||
return;
|
||||
}
|
||||
if (status == 403 || status == 405 || status == 501) {
|
||||
start_range_get(p); // HEAD refused; the ranged GET is now the primary probe
|
||||
return;
|
||||
}
|
||||
if (status >= 400) {
|
||||
deliver(p, ErrorInfo(detail::error_from_curl(CURLE_OK, status), "probe HEAD",
|
||||
static_cast<int>(status)));
|
||||
return;
|
||||
}
|
||||
// 2xx. Prove resumability with a ranged GET unless the server said "none".
|
||||
if (auto ar = p->result.accept_ranges; !ar) {
|
||||
// Accept-Ranges absent or not "bytes": still try one ranged GET — servers
|
||||
// that support ranges without advertising are common (docs/06 R4).
|
||||
}
|
||||
start_range_get(p);
|
||||
}
|
||||
|
||||
static void start_range_get(const std::shared_ptr<P> &p) {
|
||||
Request req = p->self->base_request(p->job.req);
|
||||
req.method = Method::get;
|
||||
req.range = ByteRange{0, 0};
|
||||
|
||||
TransferCallbacks cbs;
|
||||
cbs.on_head = [p](const ResponseHead &h) {
|
||||
absorb_range_head(p->result, h);
|
||||
return DataAction::abort; // we don't need the one body byte
|
||||
};
|
||||
cbs.on_data = [](ConstByteSpan) { return DataAction::abort; };
|
||||
cbs.on_finished = [p](Result<TransferStats> r) { on_range_done(p, std::move(r)); };
|
||||
|
||||
p->self->client_.start(std::move(req), std::move(cbs));
|
||||
}
|
||||
|
||||
static void absorb_range_head(ProbeResult &res, const ResponseHead &h) {
|
||||
if (h.status)
|
||||
res.http_status = h.status;
|
||||
if (!h.effective_url.empty())
|
||||
res.effective_url = h.effective_url;
|
||||
refresh_common(res, h);
|
||||
|
||||
if (h.status == 206) {
|
||||
res.accept_ranges = true; // proven, not just advertised
|
||||
if (auto cr = h.headers.get("Content-Range")) {
|
||||
if (auto total = total_from_content_range(*cr))
|
||||
res.total_size = total;
|
||||
}
|
||||
} else if (h.status == 200) {
|
||||
if (h.content_length)
|
||||
res.total_size = h.content_length;
|
||||
}
|
||||
}
|
||||
|
||||
static void on_range_done(std::shared_ptr<P> p, Result<TransferStats> r) {
|
||||
const long status = p->result.http_status;
|
||||
|
||||
if (status == 0) { // range GET died at transport level
|
||||
if (p->had_head_ok) {
|
||||
p->result.resumable = false;
|
||||
finalize_and_deliver(p);
|
||||
} else {
|
||||
deliver(p, r.has_value() ? Result<ProbeResult>(ErrorInfo(Error::probe_failed))
|
||||
: Result<ProbeResult>(std::move(r).error()));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (status == 401 || status == 407) {
|
||||
p->result.requires_auth = true;
|
||||
finalize_and_deliver(p);
|
||||
return;
|
||||
}
|
||||
if (status == 206) {
|
||||
p->result.resumable = has_validator(p->result);
|
||||
finalize_and_deliver(p);
|
||||
return;
|
||||
}
|
||||
if (status == 200 || status == 416) {
|
||||
p->result.resumable = false;
|
||||
finalize_and_deliver(p);
|
||||
return;
|
||||
}
|
||||
if (status >= 400) {
|
||||
if (p->had_head_ok) {
|
||||
p->result.resumable = false;
|
||||
finalize_and_deliver(p);
|
||||
} else {
|
||||
deliver(p, ErrorInfo(detail::error_from_curl(CURLE_OK, status), "probe ranged GET",
|
||||
static_cast<int>(status)));
|
||||
}
|
||||
return;
|
||||
}
|
||||
p->result.resumable = false;
|
||||
finalize_and_deliver(p);
|
||||
}
|
||||
|
||||
static void finalize_and_deliver(const std::shared_ptr<P> &p) {
|
||||
ProbeResult &res = p->result;
|
||||
if (res.effective_url.empty())
|
||||
res.effective_url = p->job.req.url;
|
||||
res.filename_from_url = url_filename(res.effective_url);
|
||||
if (res.filename_from_url.empty() && res.effective_url != p->job.req.url)
|
||||
res.filename_from_url = url_filename(p->job.req.url);
|
||||
|
||||
res.redirect_chain.clear();
|
||||
res.redirect_chain.push_back(p->job.req.url);
|
||||
if (res.effective_url != p->job.req.url)
|
||||
res.redirect_chain.push_back(res.effective_url);
|
||||
|
||||
deliver(p, ProbeResult(res));
|
||||
}
|
||||
|
||||
static void deliver(const std::shared_ptr<P> &p, Result<ProbeResult> out) {
|
||||
if (p->delivered)
|
||||
return;
|
||||
p->delivered = true;
|
||||
Impl *self = p->self;
|
||||
p->job.done(std::move(out));
|
||||
self->finish_one();
|
||||
}
|
||||
};
|
||||
|
||||
// --- Prober -----------------------------------------------------------------------
|
||||
Prober::Prober(unsigned max_concurrent) : impl_(std::make_unique<Impl>(max_concurrent)) {}
|
||||
Prober::~Prober() = default;
|
||||
|
||||
void Prober::probe(ProbeRequest req, std::function<void(Result<ProbeResult>)> done) {
|
||||
impl_->submit(Impl::Job{std::move(req), std::move(done)});
|
||||
}
|
||||
|
||||
} // namespace vdm::net
|
||||
Reference in New Issue
Block a user