// vdm/net/probe.cpp #include "vdm/net/probe.hpp" #include #include #include #include "net/curl_error.hpp" #include "vdm/net/http_client.hpp" #include "vdm/net/url.hpp" namespace vdm::net { namespace { std::string_view trim(std::string_view s) { while (!s.empty() && (s.front() == ' ' || s.front() == '\t')) s.remove_prefix(1); while (!s.empty() && (s.back() == ' ' || s.back() == '\t')) s.remove_suffix(1); return s; } bool iequals(std::string_view a, std::string_view b) { return HeaderList::iequals(a, b); } // "bytes 0-0/12345" -> 12345 ; "bytes 0-0/*" or malformed -> nullopt std::optional total_from_content_range(std::string_view v) { auto slash = v.find('/'); if (slash == std::string_view::npos) return std::nullopt; std::string_view tail = trim(v.substr(slash + 1)); if (tail.empty() || tail == "*") return std::nullopt; std::uint64_t n = 0; auto [p, ec] = std::from_chars(tail.data(), tail.data() + tail.size(), n); (void)p; if (ec != std::errc{}) return std::nullopt; return n; } std::string light_sanitize(std::string_view in) { std::string out; out.reserve(in.size()); for (unsigned char c : in) { if (c == '/' || c == '\\' || c == 0) { out.push_back('_'); } else if (c >= 0x20 || (c & 0x80)) { // keep printable ASCII + all UTF-8 bytes out.push_back(static_cast(c)); } } while (!out.empty() && (out.back() == '.' || out.back() == ' ')) out.pop_back(); if (out == "." || out == "..") out.clear(); return out; } } // namespace std::string suggest_filename(const ProbeResult &r, std::string_view explicit_name) { std::string cand; if (!explicit_name.empty()) cand = light_sanitize(explicit_name); if (cand.empty() && !r.filename_from_disposition.empty()) cand = light_sanitize(r.filename_from_disposition); if (cand.empty() && !r.filename_from_url.empty()) cand = light_sanitize(r.filename_from_url); if (cand.empty()) cand = "download.bin"; return cand; } // --- Prober::Impl --------------------------------------------------------------------- struct Prober::Impl { struct Job { ProbeRequest req; std::function)> done; }; struct P { Impl *self = nullptr; Job job; ProbeResult result; bool had_head_ok = false; bool delivered = false; }; explicit Impl(unsigned max_concurrent) : max_(max_concurrent ? max_concurrent : 1), client_(HttpClient::Options{.workers = 2}) {} unsigned max_; HttpClient client_; std::mutex mu_; unsigned inflight_ = 0; std::deque pending_; void submit(Job j) { { std::lock_guard lk(mu_); if (inflight_ >= max_) { pending_.push_back(std::move(j)); return; } ++inflight_; } start(std::move(j)); } void finish_one() { Job next; bool have_next = false; { std::lock_guard lk(mu_); --inflight_; if (!pending_.empty()) { next = std::move(pending_.front()); pending_.pop_front(); ++inflight_; have_next = true; } } if (have_next) start(std::move(next)); } Request base_request(const ProbeRequest &pr) { Request r; r.url = pr.url; r.headers = pr.headers; r.cookies = pr.cookies; r.user_agent = pr.user_agent; r.referrer = pr.referrer; r.proxy = pr.proxy; r.auth = pr.auth; r.follow_redirects = true; r.accept_encoding = false; r.connect_timeout_ms = pr.connect_timeout_ms; r.overall_timeout_ms = pr.overall_timeout_ms; r.low_speed_bytes_per_sec = 0; // probes are tiny; no stall detector r.low_speed_secs = 0; return r; } void start(Job j) { auto p = std::make_shared

(); p->self = this; p->job = std::move(j); p->result.effective_url = p->job.req.url; Request req = base_request(p->job.req); req.method = Method::head; TransferCallbacks cbs; cbs.on_head = [p](const ResponseHead &h) { absorb_head(p->result, h); return head_action(p, h); }; cbs.on_data = [](ConstByteSpan) { return DataAction::abort; }; cbs.on_finished = [p](Result r) { on_head_done(p, std::move(r)); }; // The worker keeps Transfer::State alive; the callbacks keep `p` alive. Storing // the Transfer in `p` would make a p -> Transfer -> State -> cbs -> p cycle. client_.start(std::move(req), std::move(cbs)); } // We want no body from a probe, so the head callback normally aborts after headers. // The exception: a 401/407 when we were handed credentials — libcurl's CURLAUTH_ANY // has to see that response before it resends with Authorization, so let this one // through (a HEAD has no body; the ranged GET's is a single byte). The final status // then lands on the next header block. static DataAction head_action(const std::shared_ptr

&p, const ResponseHead &h) { if ((h.status == 401 || h.status == 407) && p->job.req.auth.scheme != AuthScheme::none) return DataAction::proceed; return DataAction::abort; } static void absorb_head(ProbeResult &res, const ResponseHead &h) { if (h.status) res.http_status = h.status; if (!h.effective_url.empty()) res.effective_url = h.effective_url; refresh_common(res, h); if (h.content_length && !res.total_size) res.total_size = h.content_length; } static void refresh_common(ProbeResult &res, const ResponseHead &h) { if (auto v = h.headers.get("Content-Type"); v && res.mime.empty()) { std::string_view mv = *v; mv = mv.substr(0, mv.find(';')); res.mime.assign(trim(mv)); } if (auto v = h.headers.get("ETag"); v && res.etag.empty()) res.etag.assign(*v); if (auto v = h.headers.get("Last-Modified"); v && res.last_modified.empty()) res.last_modified.assign(*v); if (auto v = h.headers.get("Accept-Ranges")) { if (iequals(trim(*v), "bytes")) res.accept_ranges = true; } if (auto v = h.headers.get("Content-Disposition"); v && res.filename_from_disposition.empty()) { ContentDisposition cd = parse_content_disposition(*v); res.disposition_type = cd.type; if (cd.has_filename()) res.filename_from_disposition = cd.filename; } } static bool has_validator(const ProbeResult &r) { return !r.etag.empty() || !r.last_modified.empty(); } static void on_head_done(std::shared_ptr

p, Result r) { const long status = p->result.http_status; if (status == 0) { // never got headers -> transport failure deliver(p, r.has_value() ? Result(ErrorInfo(Error::probe_failed)) : Result(std::move(r).error())); return; } p->had_head_ok = (status >= 200 && status < 300); if (status == 401 || status == 407) { p->result.requires_auth = true; finalize_and_deliver(p); return; } if (status == 403 || status == 405 || status == 501) { start_range_get(p); // HEAD refused; the ranged GET is now the primary probe return; } if (status >= 400) { deliver(p, ErrorInfo(detail::error_from_curl(CURLE_OK, status), "probe HEAD", static_cast(status))); return; } // 2xx. Prove resumability with a ranged GET unless the server said "none". if (auto ar = p->result.accept_ranges; !ar) { // Accept-Ranges absent or not "bytes": still try one ranged GET — servers // that support ranges without advertising are common (docs/06 R4). } start_range_get(p); } static void start_range_get(const std::shared_ptr

&p) { Request req = p->self->base_request(p->job.req); req.method = Method::get; req.range = ByteRange{0, 0}; TransferCallbacks cbs; cbs.on_head = [p](const ResponseHead &h) { absorb_range_head(p->result, h); return head_action(p, h); // abort after headers, except a 401/407 with creds }; cbs.on_data = [](ConstByteSpan) { return DataAction::abort; }; cbs.on_finished = [p](Result r) { on_range_done(p, std::move(r)); }; p->self->client_.start(std::move(req), std::move(cbs)); } static void absorb_range_head(ProbeResult &res, const ResponseHead &h) { if (h.status) res.http_status = h.status; if (!h.effective_url.empty()) res.effective_url = h.effective_url; refresh_common(res, h); if (h.status == 206) { res.accept_ranges = true; // proven, not just advertised if (auto cr = h.headers.get("Content-Range")) { if (auto total = total_from_content_range(*cr)) res.total_size = total; } } else if (h.status == 200) { if (h.content_length) res.total_size = h.content_length; } } static void on_range_done(std::shared_ptr

p, Result r) { const long status = p->result.http_status; if (status == 0) { // range GET died at transport level if (p->had_head_ok) { p->result.resumable = false; finalize_and_deliver(p); } else { deliver(p, r.has_value() ? Result(ErrorInfo(Error::probe_failed)) : Result(std::move(r).error())); } return; } if (status == 401 || status == 407) { p->result.requires_auth = true; finalize_and_deliver(p); return; } if (status == 206) { p->result.resumable = has_validator(p->result); finalize_and_deliver(p); return; } if (status == 200 || status == 416) { p->result.resumable = false; finalize_and_deliver(p); return; } if (status >= 400) { if (p->had_head_ok) { p->result.resumable = false; finalize_and_deliver(p); } else { deliver(p, ErrorInfo(detail::error_from_curl(CURLE_OK, status), "probe ranged GET", static_cast(status))); } return; } p->result.resumable = false; finalize_and_deliver(p); } static void finalize_and_deliver(const std::shared_ptr

&p) { ProbeResult &res = p->result; if (res.effective_url.empty()) res.effective_url = p->job.req.url; res.filename_from_url = url_filename(res.effective_url); if (res.filename_from_url.empty() && res.effective_url != p->job.req.url) res.filename_from_url = url_filename(p->job.req.url); res.redirect_chain.clear(); res.redirect_chain.push_back(p->job.req.url); if (res.effective_url != p->job.req.url) res.redirect_chain.push_back(res.effective_url); deliver(p, ProbeResult(res)); } static void deliver(const std::shared_ptr

&p, Result out) { if (p->delivered) return; p->delivered = true; Impl *self = p->self; p->job.done(std::move(out)); self->finish_one(); } }; // --- Prober ----------------------------------------------------------------------- Prober::Prober(unsigned max_concurrent) : impl_(std::make_unique(max_concurrent)) {} Prober::~Prober() = default; void Prober::probe(ProbeRequest req, std::function)> done) { impl_->submit(Impl::Job{std::move(req), std::move(done)}); } } // namespace vdm::net