samiandClaude Sonnet 5
afaded85f8
libcurl with CURLAUTH_ANY answers a 401/407 by resending the request with an Authorization header. Two spots in net/ cut that short: - http_client's header callback delivered the response head exactly once and latched `head_delivered`, so after an auth challenge the caller only ever saw the 401 — never the 2xx of the authenticated resend. Reset the latch when a fresh status line follows a delivered 401/407 (redirects never reach that path — their head is suppressed). - the prober's head callbacks return DataAction::abort to skip the body, which also aborts the transfer mid-handshake. Return `proceed` for a 401/407 when credentials were supplied, so curl's resend can run; the real status lands on the next header block. Also give ProbeRequest an `auth` field (default scheme == none) and pass it through base_request(), so a re-probe after a 401 can present the credentials the user just entered. No behaviour change when no auth is configured. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01HPPSGhiArbvQgwC2DNiURS
361 lines
12 KiB
C++
361 lines
12 KiB
C++
// vdm/net/probe.cpp
|
|
|
|
#include "vdm/net/probe.hpp"
|
|
|
|
#include <charconv>
|
|
#include <deque>
|
|
#include <mutex>
|
|
|
|
#include "net/curl_error.hpp"
|
|
#include "vdm/net/http_client.hpp"
|
|
#include "vdm/net/url.hpp"
|
|
|
|
namespace vdm::net {
|
|
namespace {
|
|
|
|
std::string_view trim(std::string_view s) {
|
|
while (!s.empty() && (s.front() == ' ' || s.front() == '\t'))
|
|
s.remove_prefix(1);
|
|
while (!s.empty() && (s.back() == ' ' || s.back() == '\t'))
|
|
s.remove_suffix(1);
|
|
return s;
|
|
}
|
|
|
|
bool iequals(std::string_view a, std::string_view b) {
|
|
return HeaderList::iequals(a, b);
|
|
}
|
|
|
|
// "bytes 0-0/12345" -> 12345 ; "bytes 0-0/*" or malformed -> nullopt
|
|
std::optional<std::uint64_t> total_from_content_range(std::string_view v) {
|
|
auto slash = v.find('/');
|
|
if (slash == std::string_view::npos)
|
|
return std::nullopt;
|
|
std::string_view tail = trim(v.substr(slash + 1));
|
|
if (tail.empty() || tail == "*")
|
|
return std::nullopt;
|
|
std::uint64_t n = 0;
|
|
auto [p, ec] = std::from_chars(tail.data(), tail.data() + tail.size(), n);
|
|
(void)p;
|
|
if (ec != std::errc{})
|
|
return std::nullopt;
|
|
return n;
|
|
}
|
|
|
|
std::string light_sanitize(std::string_view in) {
|
|
std::string out;
|
|
out.reserve(in.size());
|
|
for (unsigned char c : in) {
|
|
if (c == '/' || c == '\\' || c == 0) {
|
|
out.push_back('_');
|
|
} else if (c >= 0x20 || (c & 0x80)) { // keep printable ASCII + all UTF-8 bytes
|
|
out.push_back(static_cast<char>(c));
|
|
}
|
|
}
|
|
while (!out.empty() && (out.back() == '.' || out.back() == ' '))
|
|
out.pop_back();
|
|
if (out == "." || out == "..")
|
|
out.clear();
|
|
return out;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
std::string suggest_filename(const ProbeResult &r, std::string_view explicit_name) {
|
|
std::string cand;
|
|
if (!explicit_name.empty())
|
|
cand = light_sanitize(explicit_name);
|
|
if (cand.empty() && !r.filename_from_disposition.empty())
|
|
cand = light_sanitize(r.filename_from_disposition);
|
|
if (cand.empty() && !r.filename_from_url.empty())
|
|
cand = light_sanitize(r.filename_from_url);
|
|
if (cand.empty())
|
|
cand = "download.bin";
|
|
return cand;
|
|
}
|
|
|
|
// --- Prober::Impl ---------------------------------------------------------------------
|
|
struct Prober::Impl {
|
|
struct Job {
|
|
ProbeRequest req;
|
|
std::function<void(Result<ProbeResult>)> done;
|
|
};
|
|
|
|
struct P {
|
|
Impl *self = nullptr;
|
|
Job job;
|
|
ProbeResult result;
|
|
bool had_head_ok = false;
|
|
bool delivered = false;
|
|
};
|
|
|
|
explicit Impl(unsigned max_concurrent)
|
|
: max_(max_concurrent ? max_concurrent : 1), client_(HttpClient::Options{.workers = 2}) {}
|
|
|
|
unsigned max_;
|
|
HttpClient client_;
|
|
std::mutex mu_;
|
|
unsigned inflight_ = 0;
|
|
std::deque<Job> pending_;
|
|
|
|
void submit(Job j) {
|
|
{
|
|
std::lock_guard lk(mu_);
|
|
if (inflight_ >= max_) {
|
|
pending_.push_back(std::move(j));
|
|
return;
|
|
}
|
|
++inflight_;
|
|
}
|
|
start(std::move(j));
|
|
}
|
|
|
|
void finish_one() {
|
|
Job next;
|
|
bool have_next = false;
|
|
{
|
|
std::lock_guard lk(mu_);
|
|
--inflight_;
|
|
if (!pending_.empty()) {
|
|
next = std::move(pending_.front());
|
|
pending_.pop_front();
|
|
++inflight_;
|
|
have_next = true;
|
|
}
|
|
}
|
|
if (have_next)
|
|
start(std::move(next));
|
|
}
|
|
|
|
Request base_request(const ProbeRequest &pr) {
|
|
Request r;
|
|
r.url = pr.url;
|
|
r.headers = pr.headers;
|
|
r.cookies = pr.cookies;
|
|
r.user_agent = pr.user_agent;
|
|
r.referrer = pr.referrer;
|
|
r.proxy = pr.proxy;
|
|
r.auth = pr.auth;
|
|
r.follow_redirects = true;
|
|
r.accept_encoding = false;
|
|
r.connect_timeout_ms = pr.connect_timeout_ms;
|
|
r.overall_timeout_ms = pr.overall_timeout_ms;
|
|
r.low_speed_bytes_per_sec = 0; // probes are tiny; no stall detector
|
|
r.low_speed_secs = 0;
|
|
return r;
|
|
}
|
|
|
|
void start(Job j) {
|
|
auto p = std::make_shared<P>();
|
|
p->self = this;
|
|
p->job = std::move(j);
|
|
p->result.effective_url = p->job.req.url;
|
|
|
|
Request req = base_request(p->job.req);
|
|
req.method = Method::head;
|
|
|
|
TransferCallbacks cbs;
|
|
cbs.on_head = [p](const ResponseHead &h) {
|
|
absorb_head(p->result, h);
|
|
return head_action(p, h);
|
|
};
|
|
cbs.on_data = [](ConstByteSpan) { return DataAction::abort; };
|
|
cbs.on_finished = [p](Result<TransferStats> r) { on_head_done(p, std::move(r)); };
|
|
|
|
// The worker keeps Transfer::State alive; the callbacks keep `p` alive. Storing
|
|
// the Transfer in `p` would make a p -> Transfer -> State -> cbs -> p cycle.
|
|
client_.start(std::move(req), std::move(cbs));
|
|
}
|
|
|
|
// We want no body from a probe, so the head callback normally aborts after headers.
|
|
// The exception: a 401/407 when we were handed credentials — libcurl's CURLAUTH_ANY
|
|
// has to see that response before it resends with Authorization, so let this one
|
|
// through (a HEAD has no body; the ranged GET's is a single byte). The final status
|
|
// then lands on the next header block.
|
|
static DataAction head_action(const std::shared_ptr<P> &p, const ResponseHead &h) {
|
|
if ((h.status == 401 || h.status == 407) && p->job.req.auth.scheme != AuthScheme::none)
|
|
return DataAction::proceed;
|
|
return DataAction::abort;
|
|
}
|
|
|
|
static void absorb_head(ProbeResult &res, const ResponseHead &h) {
|
|
if (h.status)
|
|
res.http_status = h.status;
|
|
if (!h.effective_url.empty())
|
|
res.effective_url = h.effective_url;
|
|
refresh_common(res, h);
|
|
if (h.content_length && !res.total_size)
|
|
res.total_size = h.content_length;
|
|
}
|
|
|
|
static void refresh_common(ProbeResult &res, const ResponseHead &h) {
|
|
if (auto v = h.headers.get("Content-Type"); v && res.mime.empty()) {
|
|
std::string_view mv = *v;
|
|
mv = mv.substr(0, mv.find(';'));
|
|
res.mime.assign(trim(mv));
|
|
}
|
|
if (auto v = h.headers.get("ETag"); v && res.etag.empty())
|
|
res.etag.assign(*v);
|
|
if (auto v = h.headers.get("Last-Modified"); v && res.last_modified.empty())
|
|
res.last_modified.assign(*v);
|
|
if (auto v = h.headers.get("Accept-Ranges")) {
|
|
if (iequals(trim(*v), "bytes"))
|
|
res.accept_ranges = true;
|
|
}
|
|
if (auto v = h.headers.get("Content-Disposition");
|
|
v && res.filename_from_disposition.empty()) {
|
|
ContentDisposition cd = parse_content_disposition(*v);
|
|
res.disposition_type = cd.type;
|
|
if (cd.has_filename())
|
|
res.filename_from_disposition = cd.filename;
|
|
}
|
|
}
|
|
|
|
static bool has_validator(const ProbeResult &r) {
|
|
return !r.etag.empty() || !r.last_modified.empty();
|
|
}
|
|
|
|
static void on_head_done(std::shared_ptr<P> p, Result<TransferStats> r) {
|
|
const long status = p->result.http_status;
|
|
|
|
if (status == 0) { // never got headers -> transport failure
|
|
deliver(p, r.has_value() ? Result<ProbeResult>(ErrorInfo(Error::probe_failed))
|
|
: Result<ProbeResult>(std::move(r).error()));
|
|
return;
|
|
}
|
|
p->had_head_ok = (status >= 200 && status < 300);
|
|
|
|
if (status == 401 || status == 407) {
|
|
p->result.requires_auth = true;
|
|
finalize_and_deliver(p);
|
|
return;
|
|
}
|
|
if (status == 403 || status == 405 || status == 501) {
|
|
start_range_get(p); // HEAD refused; the ranged GET is now the primary probe
|
|
return;
|
|
}
|
|
if (status >= 400) {
|
|
deliver(p, ErrorInfo(detail::error_from_curl(CURLE_OK, status), "probe HEAD",
|
|
static_cast<int>(status)));
|
|
return;
|
|
}
|
|
// 2xx. Prove resumability with a ranged GET unless the server said "none".
|
|
if (auto ar = p->result.accept_ranges; !ar) {
|
|
// Accept-Ranges absent or not "bytes": still try one ranged GET — servers
|
|
// that support ranges without advertising are common (docs/06 R4).
|
|
}
|
|
start_range_get(p);
|
|
}
|
|
|
|
static void start_range_get(const std::shared_ptr<P> &p) {
|
|
Request req = p->self->base_request(p->job.req);
|
|
req.method = Method::get;
|
|
req.range = ByteRange{0, 0};
|
|
|
|
TransferCallbacks cbs;
|
|
cbs.on_head = [p](const ResponseHead &h) {
|
|
absorb_range_head(p->result, h);
|
|
return head_action(p, h); // abort after headers, except a 401/407 with creds
|
|
};
|
|
cbs.on_data = [](ConstByteSpan) { return DataAction::abort; };
|
|
cbs.on_finished = [p](Result<TransferStats> r) { on_range_done(p, std::move(r)); };
|
|
|
|
p->self->client_.start(std::move(req), std::move(cbs));
|
|
}
|
|
|
|
static void absorb_range_head(ProbeResult &res, const ResponseHead &h) {
|
|
if (h.status)
|
|
res.http_status = h.status;
|
|
if (!h.effective_url.empty())
|
|
res.effective_url = h.effective_url;
|
|
refresh_common(res, h);
|
|
|
|
if (h.status == 206) {
|
|
res.accept_ranges = true; // proven, not just advertised
|
|
if (auto cr = h.headers.get("Content-Range")) {
|
|
if (auto total = total_from_content_range(*cr))
|
|
res.total_size = total;
|
|
}
|
|
} else if (h.status == 200) {
|
|
if (h.content_length)
|
|
res.total_size = h.content_length;
|
|
}
|
|
}
|
|
|
|
static void on_range_done(std::shared_ptr<P> p, Result<TransferStats> r) {
|
|
const long status = p->result.http_status;
|
|
|
|
if (status == 0) { // range GET died at transport level
|
|
if (p->had_head_ok) {
|
|
p->result.resumable = false;
|
|
finalize_and_deliver(p);
|
|
} else {
|
|
deliver(p, r.has_value() ? Result<ProbeResult>(ErrorInfo(Error::probe_failed))
|
|
: Result<ProbeResult>(std::move(r).error()));
|
|
}
|
|
return;
|
|
}
|
|
if (status == 401 || status == 407) {
|
|
p->result.requires_auth = true;
|
|
finalize_and_deliver(p);
|
|
return;
|
|
}
|
|
if (status == 206) {
|
|
p->result.resumable = has_validator(p->result);
|
|
finalize_and_deliver(p);
|
|
return;
|
|
}
|
|
if (status == 200 || status == 416) {
|
|
p->result.resumable = false;
|
|
finalize_and_deliver(p);
|
|
return;
|
|
}
|
|
if (status >= 400) {
|
|
if (p->had_head_ok) {
|
|
p->result.resumable = false;
|
|
finalize_and_deliver(p);
|
|
} else {
|
|
deliver(p, ErrorInfo(detail::error_from_curl(CURLE_OK, status), "probe ranged GET",
|
|
static_cast<int>(status)));
|
|
}
|
|
return;
|
|
}
|
|
p->result.resumable = false;
|
|
finalize_and_deliver(p);
|
|
}
|
|
|
|
static void finalize_and_deliver(const std::shared_ptr<P> &p) {
|
|
ProbeResult &res = p->result;
|
|
if (res.effective_url.empty())
|
|
res.effective_url = p->job.req.url;
|
|
res.filename_from_url = url_filename(res.effective_url);
|
|
if (res.filename_from_url.empty() && res.effective_url != p->job.req.url)
|
|
res.filename_from_url = url_filename(p->job.req.url);
|
|
|
|
res.redirect_chain.clear();
|
|
res.redirect_chain.push_back(p->job.req.url);
|
|
if (res.effective_url != p->job.req.url)
|
|
res.redirect_chain.push_back(res.effective_url);
|
|
|
|
deliver(p, ProbeResult(res));
|
|
}
|
|
|
|
static void deliver(const std::shared_ptr<P> &p, Result<ProbeResult> out) {
|
|
if (p->delivered)
|
|
return;
|
|
p->delivered = true;
|
|
Impl *self = p->self;
|
|
p->job.done(std::move(out));
|
|
self->finish_one();
|
|
}
|
|
};
|
|
|
|
// --- Prober -----------------------------------------------------------------------
|
|
Prober::Prober(unsigned max_concurrent) : impl_(std::make_unique<Impl>(max_concurrent)) {}
|
|
Prober::~Prober() = default;
|
|
|
|
void Prober::probe(ProbeRequest req, std::function<void(Result<ProbeResult>)> done) {
|
|
impl_->submit(Impl::Job{std::move(req), std::move(done)});
|
|
}
|
|
|
|
} // namespace vdm::net
|