core: net/probe + Content-Disposition parser + URL splitter (stage 3)
net/content_disposition — total parser for the mojibake-prone header: RFC 6266 filename (quoted/token), RFC 5987 filename* ext-values (charset'lang'pct-encoded, incl. RFC 2231 continuations), legacy RFC 2047 encoded-words (=?UTF-8?B?..?= / ?Q?), and raw Latin-1 bytes; prefers filename* over filename; strips path components AFTER decoding (a base64 payload can hold '/'). 22-case test table. net/text_codec (internal) — percent-decode, UTF-8 validation, Latin-1-> UTF-8, base64, RFC 2047 — shared by the CD parser and the URL splitter. net/url — a small total URL splitter (scheme/userinfo/host/port/path/ query/fragment, http(s) validity) and url_filename() for the last path segment; used for the filename fallback. net/probe — HEAD then a ranged GET bytes=0-0 that PROVES resumability (206 + matching Content-Range + a validator), rather than trusting Accept-Ranges which servers lie about; the ranged GET is also the HEAD- refused (403/405/501) fallback. 401/407 -> success result with requires_auth, not an error. Runs on its own pool (max_concurrent, default 4) outside the segment budget per ADR 0011 §5. suggest_filename() does the resolution order (explicit -> disposition -> URL -> download.bin) with a light strip; rules/ (stage 9) owns the authoritative sanitize. tools/fuzz — libFuzzer targets for the CD parser and the URL splitter, compiling the parser sources directly so they're fully instrumented; self-guards on VELOX_BUILD_FUZZ + Clang (the top-level CMake adds every tools/* unconditionally). Seed corpora included. Fixed on the way: a p -> Transfer -> State -> cbs -> p reference cycle in Prober that leaked every probe (drop the stored Transfer; the worker keeps State alive). Tests green under ASan/UBSan and TSan. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01HPPSGhiArbvQgwC2DNiURS
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,279 @@
|
||||
// vdm/net/content_disposition.cpp
|
||||
|
||||
#include "vdm/net/content_disposition.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "net/text_codec.hpp"
|
||||
|
||||
namespace vdm::net {
|
||||
namespace {
|
||||
|
||||
using detail::decode_rfc2047;
|
||||
using detail::latin1_to_utf8;
|
||||
using detail::percent_decode;
|
||||
using detail::to_utf8_best_effort;
|
||||
|
||||
std::string_view trim(std::string_view s) {
|
||||
while (!s.empty() && (s.front() == ' ' || s.front() == '\t'))
|
||||
s.remove_prefix(1);
|
||||
while (!s.empty() &&
|
||||
(s.back() == ' ' || s.back() == '\t' || s.back() == '\r' || s.back() == '\n'))
|
||||
s.remove_suffix(1);
|
||||
return s;
|
||||
}
|
||||
|
||||
std::string ascii_lower(std::string_view s) {
|
||||
std::string r(s);
|
||||
for (char &c : r)
|
||||
if (c >= 'A' && c <= 'Z')
|
||||
c = static_cast<char>(c - 'A' + 'a');
|
||||
return r;
|
||||
}
|
||||
|
||||
// Split "type; a=b; c*=d; e=\"f;g\"" into the type and a param list, honouring quoted
|
||||
// strings (a ';' inside quotes is not a separator). For quoted values the value stored is
|
||||
// the *raw* inner text (quotes removed, escapes NOT yet resolved) with `quoted = true`;
|
||||
// callers resolve escapes and strip path components together (order matters — see
|
||||
// unquote_strip). Keys are lowercased.
|
||||
struct Param {
|
||||
std::string key;
|
||||
std::string value;
|
||||
bool quoted = false;
|
||||
};
|
||||
struct Params {
|
||||
std::string type;
|
||||
std::vector<Param> kv;
|
||||
|
||||
[[nodiscard]] const Param *find(std::string_view key) const {
|
||||
for (const auto &p : kv)
|
||||
if (p.key == key)
|
||||
return &p;
|
||||
return nullptr;
|
||||
}
|
||||
};
|
||||
|
||||
Params tokenize(std::string_view h) {
|
||||
Params out;
|
||||
std::size_t i = 0;
|
||||
const std::size_t n = h.size();
|
||||
|
||||
auto read_segment = [&]() -> std::string_view {
|
||||
std::size_t start = i;
|
||||
bool in_q = false;
|
||||
for (; i < n; ++i) {
|
||||
char c = h[i];
|
||||
if (c == '"') {
|
||||
in_q = !in_q;
|
||||
} else if (c == '\\' && in_q && i + 1 < n) {
|
||||
++i; // skip escaped char
|
||||
} else if (c == ';' && !in_q) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
std::string_view seg = h.substr(start, i - start);
|
||||
if (i < n)
|
||||
++i; // consume ';'
|
||||
return seg;
|
||||
};
|
||||
|
||||
out.type = ascii_lower(trim(read_segment()));
|
||||
|
||||
while (i < n) {
|
||||
std::string_view seg = trim(read_segment());
|
||||
if (seg.empty())
|
||||
continue;
|
||||
auto eq = seg.find('=');
|
||||
if (eq == std::string_view::npos) {
|
||||
out.kv.emplace_back(ascii_lower(seg), std::string{});
|
||||
continue;
|
||||
}
|
||||
std::string key = ascii_lower(trim(seg.substr(0, eq)));
|
||||
std::string_view rawval = trim(seg.substr(eq + 1));
|
||||
|
||||
Param param;
|
||||
param.key = std::move(key);
|
||||
if (rawval.size() >= 2 && rawval.front() == '"') {
|
||||
std::string_view inner = rawval.substr(1);
|
||||
auto close = inner.rfind('"');
|
||||
if (close != std::string_view::npos)
|
||||
inner = inner.substr(0, close);
|
||||
param.value.assign(inner); // raw, escapes unresolved
|
||||
param.quoted = true;
|
||||
} else {
|
||||
param.value.assign(rawval);
|
||||
}
|
||||
out.kv.push_back(std::move(param));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string strip_path(std::string s) {
|
||||
auto slash = s.find_last_of("/\\");
|
||||
if (slash != std::string::npos)
|
||||
s.erase(0, slash + 1);
|
||||
return s;
|
||||
}
|
||||
|
||||
// Resolve ONLY the `\"` escape (needed so a quote can appear mid-name). Every other
|
||||
// backslash is kept literal and later treated as a path separator by strip_path — real
|
||||
// Windows paths in the wild use `\` unescaped, and path-traversal defence matters more
|
||||
// than supporting the vanishingly rare filename with a literal backslash.
|
||||
std::string unescape_dquote(std::string_view raw) {
|
||||
std::string out;
|
||||
out.reserve(raw.size());
|
||||
for (std::size_t i = 0; i < raw.size(); ++i) {
|
||||
if (raw[i] == '\\' && i + 1 < raw.size() && raw[i + 1] == '"') {
|
||||
out.push_back('"');
|
||||
++i;
|
||||
} else {
|
||||
out.push_back(raw[i]);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Decode an RFC 5987 ext-value: charset'lang'pct-encoded-octets
|
||||
std::string decode_ext_value(std::string_view v) {
|
||||
auto q1 = v.find('\'');
|
||||
if (q1 == std::string_view::npos)
|
||||
return percent_decode(v); // malformed: best effort
|
||||
auto q2 = v.find('\'', q1 + 1);
|
||||
if (q2 == std::string_view::npos)
|
||||
return percent_decode(v.substr(q1 + 1));
|
||||
|
||||
std::string_view charset = v.substr(0, q1);
|
||||
std::string_view enc = v.substr(q2 + 1);
|
||||
std::string bytes = percent_decode(enc);
|
||||
|
||||
std::string cs = ascii_lower(charset);
|
||||
if (cs == "iso-8859-1" || cs == "latin1")
|
||||
return latin1_to_utf8(bytes);
|
||||
return to_utf8_best_effort(bytes); // utf-8 or unknown -> best effort
|
||||
}
|
||||
|
||||
// Reassemble RFC 2231 continuations: name*0*, name*1, name*2* ... in order.
|
||||
std::string join_continuations(const Params &p, std::string_view base, bool &is_ext) {
|
||||
std::vector<std::pair<int, std::string>> parts;
|
||||
is_ext = false;
|
||||
for (const auto ¶m : p.kv) {
|
||||
const std::string &k = param.key;
|
||||
const std::string &val = param.value;
|
||||
if (k.size() <= base.size() + 1 || k.compare(0, base.size(), base) != 0)
|
||||
continue;
|
||||
if (k[base.size()] != '*')
|
||||
continue;
|
||||
std::string_view rest(k);
|
||||
rest.remove_prefix(base.size() + 1); // after "base*"
|
||||
bool star = false;
|
||||
if (!rest.empty() && rest.back() == '*') {
|
||||
star = true;
|
||||
rest.remove_suffix(1);
|
||||
}
|
||||
int idx = 0;
|
||||
for (char c : rest) {
|
||||
if (c < '0' || c > '9') {
|
||||
idx = -1;
|
||||
break;
|
||||
}
|
||||
idx = idx * 10 + (c - '0');
|
||||
}
|
||||
if (idx < 0)
|
||||
continue;
|
||||
if (star)
|
||||
is_ext = true;
|
||||
parts.emplace_back(idx, val);
|
||||
}
|
||||
if (parts.empty())
|
||||
return {};
|
||||
std::sort(parts.begin(), parts.end(),
|
||||
[](const auto &a, const auto &b) { return a.first < b.first; });
|
||||
|
||||
// Piece 0 (if star-form) carries charset'lang' prefix; later pieces are raw
|
||||
// percent-encoded. Concatenate the percent-encoded text then decode once.
|
||||
std::string charset_prefix;
|
||||
std::string enc;
|
||||
bool first = true;
|
||||
for (auto &[idx, val] : parts) {
|
||||
if (first && is_ext) {
|
||||
auto q1 = val.find('\'');
|
||||
auto q2 = (q1 == std::string::npos) ? std::string::npos : val.find('\'', q1 + 1);
|
||||
if (q2 != std::string::npos) {
|
||||
charset_prefix = val.substr(0, q2 + 1);
|
||||
enc += val.substr(q2 + 1);
|
||||
} else {
|
||||
enc += val;
|
||||
}
|
||||
} else {
|
||||
enc += val;
|
||||
}
|
||||
first = false;
|
||||
}
|
||||
if (is_ext)
|
||||
return decode_ext_value(charset_prefix + enc);
|
||||
return to_utf8_best_effort(percent_decode(enc));
|
||||
}
|
||||
|
||||
ContentDisposition::Type classify(std::string_view t) {
|
||||
if (t == "inline")
|
||||
return ContentDisposition::Type::inline_;
|
||||
if (t == "attachment")
|
||||
return ContentDisposition::Type::attachment;
|
||||
if (t == "form-data")
|
||||
return ContentDisposition::Type::form_data;
|
||||
if (t.empty())
|
||||
return ContentDisposition::Type::none;
|
||||
return ContentDisposition::Type::other;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
ContentDisposition parse_content_disposition(std::string_view header_value) {
|
||||
ContentDisposition cd;
|
||||
header_value = trim(header_value);
|
||||
if (header_value.empty())
|
||||
return cd;
|
||||
|
||||
Params p = tokenize(header_value);
|
||||
cd.type = classify(p.type);
|
||||
|
||||
// RFC 6266 §4.3: prefer filename* over filename.
|
||||
std::string ext_name;
|
||||
bool ext_is_ext = false;
|
||||
if (const Param *fstar = p.find("filename*")) {
|
||||
ext_name = decode_ext_value(fstar->value);
|
||||
ext_is_ext = true;
|
||||
} else {
|
||||
std::string joined = join_continuations(p, "filename", ext_is_ext);
|
||||
if (!joined.empty())
|
||||
ext_name = std::move(joined);
|
||||
}
|
||||
|
||||
std::string plain_name;
|
||||
if (const Param *f = p.find("filename")) {
|
||||
// Resolve `\"`, then decode legacy encoded-words if present — BEFORE stripping
|
||||
// path components, since a base64 payload can legitimately contain '/'.
|
||||
std::string raw = f->quoted ? unescape_dquote(f->value) : f->value;
|
||||
bool had_ew = false;
|
||||
std::string decoded = decode_rfc2047(raw, &had_ew);
|
||||
plain_name = had_ew ? std::move(decoded) : to_utf8_best_effort(raw);
|
||||
}
|
||||
|
||||
if (!ext_name.empty()) {
|
||||
cd.filename = strip_path(std::move(ext_name));
|
||||
cd.filename_from_ext = ext_is_ext;
|
||||
} else if (!plain_name.empty()) {
|
||||
cd.filename = strip_path(std::move(plain_name));
|
||||
cd.filename_from_ext = false;
|
||||
}
|
||||
|
||||
// Trim ASCII whitespace the decoders may have left at the edges.
|
||||
std::string_view fv = trim(cd.filename);
|
||||
cd.filename.assign(fv);
|
||||
return cd;
|
||||
}
|
||||
|
||||
} // namespace vdm::net
|
||||
@@ -0,0 +1,348 @@
|
||||
// vdm/net/probe.cpp
|
||||
|
||||
#include "vdm/net/probe.hpp"
|
||||
|
||||
#include <charconv>
|
||||
#include <deque>
|
||||
#include <mutex>
|
||||
|
||||
#include "net/curl_error.hpp"
|
||||
#include "vdm/net/http_client.hpp"
|
||||
#include "vdm/net/url.hpp"
|
||||
|
||||
namespace vdm::net {
|
||||
namespace {
|
||||
|
||||
std::string_view trim(std::string_view s) {
|
||||
while (!s.empty() && (s.front() == ' ' || s.front() == '\t'))
|
||||
s.remove_prefix(1);
|
||||
while (!s.empty() && (s.back() == ' ' || s.back() == '\t'))
|
||||
s.remove_suffix(1);
|
||||
return s;
|
||||
}
|
||||
|
||||
bool iequals(std::string_view a, std::string_view b) {
|
||||
return HeaderList::iequals(a, b);
|
||||
}
|
||||
|
||||
// "bytes 0-0/12345" -> 12345 ; "bytes 0-0/*" or malformed -> nullopt
|
||||
std::optional<std::uint64_t> total_from_content_range(std::string_view v) {
|
||||
auto slash = v.find('/');
|
||||
if (slash == std::string_view::npos)
|
||||
return std::nullopt;
|
||||
std::string_view tail = trim(v.substr(slash + 1));
|
||||
if (tail.empty() || tail == "*")
|
||||
return std::nullopt;
|
||||
std::uint64_t n = 0;
|
||||
auto [p, ec] = std::from_chars(tail.data(), tail.data() + tail.size(), n);
|
||||
(void)p;
|
||||
if (ec != std::errc{})
|
||||
return std::nullopt;
|
||||
return n;
|
||||
}
|
||||
|
||||
std::string light_sanitize(std::string_view in) {
|
||||
std::string out;
|
||||
out.reserve(in.size());
|
||||
for (unsigned char c : in) {
|
||||
if (c == '/' || c == '\\' || c == 0) {
|
||||
out.push_back('_');
|
||||
} else if (c >= 0x20 || (c & 0x80)) { // keep printable ASCII + all UTF-8 bytes
|
||||
out.push_back(static_cast<char>(c));
|
||||
}
|
||||
}
|
||||
while (!out.empty() && (out.back() == '.' || out.back() == ' '))
|
||||
out.pop_back();
|
||||
if (out == "." || out == "..")
|
||||
out.clear();
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string suggest_filename(const ProbeResult &r, std::string_view explicit_name) {
|
||||
std::string cand;
|
||||
if (!explicit_name.empty())
|
||||
cand = light_sanitize(explicit_name);
|
||||
if (cand.empty() && !r.filename_from_disposition.empty())
|
||||
cand = light_sanitize(r.filename_from_disposition);
|
||||
if (cand.empty() && !r.filename_from_url.empty())
|
||||
cand = light_sanitize(r.filename_from_url);
|
||||
if (cand.empty())
|
||||
cand = "download.bin";
|
||||
return cand;
|
||||
}
|
||||
|
||||
// --- Prober::Impl ---------------------------------------------------------------------
|
||||
struct Prober::Impl {
|
||||
struct Job {
|
||||
ProbeRequest req;
|
||||
std::function<void(Result<ProbeResult>)> done;
|
||||
};
|
||||
|
||||
struct P {
|
||||
Impl *self = nullptr;
|
||||
Job job;
|
||||
ProbeResult result;
|
||||
bool had_head_ok = false;
|
||||
bool delivered = false;
|
||||
};
|
||||
|
||||
explicit Impl(unsigned max_concurrent)
|
||||
: max_(max_concurrent ? max_concurrent : 1), client_(HttpClient::Options{.workers = 2}) {}
|
||||
|
||||
unsigned max_;
|
||||
HttpClient client_;
|
||||
std::mutex mu_;
|
||||
unsigned inflight_ = 0;
|
||||
std::deque<Job> pending_;
|
||||
|
||||
void submit(Job j) {
|
||||
{
|
||||
std::lock_guard lk(mu_);
|
||||
if (inflight_ >= max_) {
|
||||
pending_.push_back(std::move(j));
|
||||
return;
|
||||
}
|
||||
++inflight_;
|
||||
}
|
||||
start(std::move(j));
|
||||
}
|
||||
|
||||
void finish_one() {
|
||||
Job next;
|
||||
bool have_next = false;
|
||||
{
|
||||
std::lock_guard lk(mu_);
|
||||
--inflight_;
|
||||
if (!pending_.empty()) {
|
||||
next = std::move(pending_.front());
|
||||
pending_.pop_front();
|
||||
++inflight_;
|
||||
have_next = true;
|
||||
}
|
||||
}
|
||||
if (have_next)
|
||||
start(std::move(next));
|
||||
}
|
||||
|
||||
Request base_request(const ProbeRequest &pr) {
|
||||
Request r;
|
||||
r.url = pr.url;
|
||||
r.headers = pr.headers;
|
||||
r.cookies = pr.cookies;
|
||||
r.user_agent = pr.user_agent;
|
||||
r.referrer = pr.referrer;
|
||||
r.proxy = pr.proxy;
|
||||
r.follow_redirects = true;
|
||||
r.accept_encoding = false;
|
||||
r.connect_timeout_ms = pr.connect_timeout_ms;
|
||||
r.overall_timeout_ms = pr.overall_timeout_ms;
|
||||
r.low_speed_bytes_per_sec = 0; // probes are tiny; no stall detector
|
||||
r.low_speed_secs = 0;
|
||||
return r;
|
||||
}
|
||||
|
||||
void start(Job j) {
|
||||
auto p = std::make_shared<P>();
|
||||
p->self = this;
|
||||
p->job = std::move(j);
|
||||
p->result.effective_url = p->job.req.url;
|
||||
|
||||
Request req = base_request(p->job.req);
|
||||
req.method = Method::head;
|
||||
|
||||
TransferCallbacks cbs;
|
||||
cbs.on_head = [p](const ResponseHead &h) {
|
||||
absorb_head(p->result, h);
|
||||
return DataAction::abort;
|
||||
};
|
||||
cbs.on_data = [](ConstByteSpan) { return DataAction::abort; };
|
||||
cbs.on_finished = [p](Result<TransferStats> r) { on_head_done(p, std::move(r)); };
|
||||
|
||||
// The worker keeps Transfer::State alive; the callbacks keep `p` alive. Storing
|
||||
// the Transfer in `p` would make a p -> Transfer -> State -> cbs -> p cycle.
|
||||
client_.start(std::move(req), std::move(cbs));
|
||||
}
|
||||
|
||||
static void absorb_head(ProbeResult &res, const ResponseHead &h) {
|
||||
if (h.status)
|
||||
res.http_status = h.status;
|
||||
if (!h.effective_url.empty())
|
||||
res.effective_url = h.effective_url;
|
||||
refresh_common(res, h);
|
||||
if (h.content_length && !res.total_size)
|
||||
res.total_size = h.content_length;
|
||||
}
|
||||
|
||||
static void refresh_common(ProbeResult &res, const ResponseHead &h) {
|
||||
if (auto v = h.headers.get("Content-Type"); v && res.mime.empty()) {
|
||||
std::string_view mv = *v;
|
||||
mv = mv.substr(0, mv.find(';'));
|
||||
res.mime.assign(trim(mv));
|
||||
}
|
||||
if (auto v = h.headers.get("ETag"); v && res.etag.empty())
|
||||
res.etag.assign(*v);
|
||||
if (auto v = h.headers.get("Last-Modified"); v && res.last_modified.empty())
|
||||
res.last_modified.assign(*v);
|
||||
if (auto v = h.headers.get("Accept-Ranges")) {
|
||||
if (iequals(trim(*v), "bytes"))
|
||||
res.accept_ranges = true;
|
||||
}
|
||||
if (auto v = h.headers.get("Content-Disposition");
|
||||
v && res.filename_from_disposition.empty()) {
|
||||
ContentDisposition cd = parse_content_disposition(*v);
|
||||
res.disposition_type = cd.type;
|
||||
if (cd.has_filename())
|
||||
res.filename_from_disposition = cd.filename;
|
||||
}
|
||||
}
|
||||
|
||||
static bool has_validator(const ProbeResult &r) {
|
||||
return !r.etag.empty() || !r.last_modified.empty();
|
||||
}
|
||||
|
||||
static void on_head_done(std::shared_ptr<P> p, Result<TransferStats> r) {
|
||||
const long status = p->result.http_status;
|
||||
|
||||
if (status == 0) { // never got headers -> transport failure
|
||||
deliver(p, r.has_value() ? Result<ProbeResult>(ErrorInfo(Error::probe_failed))
|
||||
: Result<ProbeResult>(std::move(r).error()));
|
||||
return;
|
||||
}
|
||||
p->had_head_ok = (status >= 200 && status < 300);
|
||||
|
||||
if (status == 401 || status == 407) {
|
||||
p->result.requires_auth = true;
|
||||
finalize_and_deliver(p);
|
||||
return;
|
||||
}
|
||||
if (status == 403 || status == 405 || status == 501) {
|
||||
start_range_get(p); // HEAD refused; the ranged GET is now the primary probe
|
||||
return;
|
||||
}
|
||||
if (status >= 400) {
|
||||
deliver(p, ErrorInfo(detail::error_from_curl(CURLE_OK, status), "probe HEAD",
|
||||
static_cast<int>(status)));
|
||||
return;
|
||||
}
|
||||
// 2xx. Prove resumability with a ranged GET unless the server said "none".
|
||||
if (auto ar = p->result.accept_ranges; !ar) {
|
||||
// Accept-Ranges absent or not "bytes": still try one ranged GET — servers
|
||||
// that support ranges without advertising are common (docs/06 R4).
|
||||
}
|
||||
start_range_get(p);
|
||||
}
|
||||
|
||||
static void start_range_get(const std::shared_ptr<P> &p) {
|
||||
Request req = p->self->base_request(p->job.req);
|
||||
req.method = Method::get;
|
||||
req.range = ByteRange{0, 0};
|
||||
|
||||
TransferCallbacks cbs;
|
||||
cbs.on_head = [p](const ResponseHead &h) {
|
||||
absorb_range_head(p->result, h);
|
||||
return DataAction::abort; // we don't need the one body byte
|
||||
};
|
||||
cbs.on_data = [](ConstByteSpan) { return DataAction::abort; };
|
||||
cbs.on_finished = [p](Result<TransferStats> r) { on_range_done(p, std::move(r)); };
|
||||
|
||||
p->self->client_.start(std::move(req), std::move(cbs));
|
||||
}
|
||||
|
||||
static void absorb_range_head(ProbeResult &res, const ResponseHead &h) {
|
||||
if (h.status)
|
||||
res.http_status = h.status;
|
||||
if (!h.effective_url.empty())
|
||||
res.effective_url = h.effective_url;
|
||||
refresh_common(res, h);
|
||||
|
||||
if (h.status == 206) {
|
||||
res.accept_ranges = true; // proven, not just advertised
|
||||
if (auto cr = h.headers.get("Content-Range")) {
|
||||
if (auto total = total_from_content_range(*cr))
|
||||
res.total_size = total;
|
||||
}
|
||||
} else if (h.status == 200) {
|
||||
if (h.content_length)
|
||||
res.total_size = h.content_length;
|
||||
}
|
||||
}
|
||||
|
||||
static void on_range_done(std::shared_ptr<P> p, Result<TransferStats> r) {
|
||||
const long status = p->result.http_status;
|
||||
|
||||
if (status == 0) { // range GET died at transport level
|
||||
if (p->had_head_ok) {
|
||||
p->result.resumable = false;
|
||||
finalize_and_deliver(p);
|
||||
} else {
|
||||
deliver(p, r.has_value() ? Result<ProbeResult>(ErrorInfo(Error::probe_failed))
|
||||
: Result<ProbeResult>(std::move(r).error()));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (status == 401 || status == 407) {
|
||||
p->result.requires_auth = true;
|
||||
finalize_and_deliver(p);
|
||||
return;
|
||||
}
|
||||
if (status == 206) {
|
||||
p->result.resumable = has_validator(p->result);
|
||||
finalize_and_deliver(p);
|
||||
return;
|
||||
}
|
||||
if (status == 200 || status == 416) {
|
||||
p->result.resumable = false;
|
||||
finalize_and_deliver(p);
|
||||
return;
|
||||
}
|
||||
if (status >= 400) {
|
||||
if (p->had_head_ok) {
|
||||
p->result.resumable = false;
|
||||
finalize_and_deliver(p);
|
||||
} else {
|
||||
deliver(p, ErrorInfo(detail::error_from_curl(CURLE_OK, status), "probe ranged GET",
|
||||
static_cast<int>(status)));
|
||||
}
|
||||
return;
|
||||
}
|
||||
p->result.resumable = false;
|
||||
finalize_and_deliver(p);
|
||||
}
|
||||
|
||||
static void finalize_and_deliver(const std::shared_ptr<P> &p) {
|
||||
ProbeResult &res = p->result;
|
||||
if (res.effective_url.empty())
|
||||
res.effective_url = p->job.req.url;
|
||||
res.filename_from_url = url_filename(res.effective_url);
|
||||
if (res.filename_from_url.empty() && res.effective_url != p->job.req.url)
|
||||
res.filename_from_url = url_filename(p->job.req.url);
|
||||
|
||||
res.redirect_chain.clear();
|
||||
res.redirect_chain.push_back(p->job.req.url);
|
||||
if (res.effective_url != p->job.req.url)
|
||||
res.redirect_chain.push_back(res.effective_url);
|
||||
|
||||
deliver(p, ProbeResult(res));
|
||||
}
|
||||
|
||||
static void deliver(const std::shared_ptr<P> &p, Result<ProbeResult> out) {
|
||||
if (p->delivered)
|
||||
return;
|
||||
p->delivered = true;
|
||||
Impl *self = p->self;
|
||||
p->job.done(std::move(out));
|
||||
self->finish_one();
|
||||
}
|
||||
};
|
||||
|
||||
// --- Prober -----------------------------------------------------------------------
|
||||
Prober::Prober(unsigned max_concurrent) : impl_(std::make_unique<Impl>(max_concurrent)) {}
|
||||
Prober::~Prober() = default;
|
||||
|
||||
void Prober::probe(ProbeRequest req, std::function<void(Result<ProbeResult>)> done) {
|
||||
impl_->submit(Impl::Job{std::move(req), std::move(done)});
|
||||
}
|
||||
|
||||
} // namespace vdm::net
|
||||
@@ -0,0 +1,262 @@
|
||||
// vdm/net/text_codec.cpp
|
||||
|
||||
#include "net/text_codec.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <cctype>
|
||||
|
||||
namespace vdm::net::detail {
|
||||
namespace {
|
||||
|
||||
int hex_val(char c) noexcept {
|
||||
if (c >= '0' && c <= '9')
|
||||
return c - '0';
|
||||
if (c >= 'a' && c <= 'f')
|
||||
return c - 'a' + 10;
|
||||
if (c >= 'A' && c <= 'F')
|
||||
return c - 'A' + 10;
|
||||
return -1;
|
||||
}
|
||||
|
||||
void append_utf8(std::string &out, std::uint32_t cp) {
|
||||
if (cp <= 0x7F) {
|
||||
out.push_back(static_cast<char>(cp));
|
||||
} else if (cp <= 0x7FF) {
|
||||
out.push_back(static_cast<char>(0xC0 | (cp >> 6)));
|
||||
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
|
||||
} else if (cp <= 0xFFFF) {
|
||||
out.push_back(static_cast<char>(0xE0 | (cp >> 12)));
|
||||
out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
|
||||
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
|
||||
} else {
|
||||
out.push_back(static_cast<char>(0xF0 | (cp >> 18)));
|
||||
out.push_back(static_cast<char>(0x80 | ((cp >> 12) & 0x3F)));
|
||||
out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
|
||||
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
|
||||
}
|
||||
}
|
||||
|
||||
bool charset_is(std::string_view cs, std::string_view want) {
|
||||
if (cs.size() != want.size())
|
||||
return false;
|
||||
for (std::size_t i = 0; i < cs.size(); ++i) {
|
||||
char a = cs[i], b = want[i];
|
||||
if (a >= 'A' && a <= 'Z')
|
||||
a = static_cast<char>(a - 'A' + 'a');
|
||||
if (b >= 'A' && b <= 'Z')
|
||||
b = static_cast<char>(b - 'A' + 'a');
|
||||
if (a != b)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool is_utf8(std::string_view cs) {
|
||||
return charset_is(cs, "utf-8") || charset_is(cs, "utf8");
|
||||
}
|
||||
bool is_latin1(std::string_view cs) {
|
||||
return charset_is(cs, "iso-8859-1") || charset_is(cs, "latin1") ||
|
||||
charset_is(cs, "iso8859-1") || charset_is(cs, "windows-1252");
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string percent_decode(std::string_view in, bool plus_as_space) {
|
||||
std::string out;
|
||||
out.reserve(in.size());
|
||||
for (std::size_t i = 0; i < in.size(); ++i) {
|
||||
char c = in[i];
|
||||
if (c == '%' && i + 2 < in.size()) {
|
||||
int hi = hex_val(in[i + 1]);
|
||||
int lo = hex_val(in[i + 2]);
|
||||
if (hi >= 0 && lo >= 0) {
|
||||
out.push_back(static_cast<char>((hi << 4) | lo));
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (c == '+' && plus_as_space) {
|
||||
out.push_back(' ');
|
||||
continue;
|
||||
}
|
||||
out.push_back(c);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
bool is_valid_utf8(std::string_view s) noexcept {
|
||||
std::size_t i = 0;
|
||||
const std::size_t n = s.size();
|
||||
auto cont = [&](std::size_t k) {
|
||||
return k < n && (static_cast<unsigned char>(s[k]) & 0xC0) == 0x80;
|
||||
};
|
||||
while (i < n) {
|
||||
unsigned char c = static_cast<unsigned char>(s[i]);
|
||||
if (c < 0x80) {
|
||||
++i;
|
||||
} else if ((c & 0xE0) == 0xC0) {
|
||||
if (!cont(i + 1))
|
||||
return false;
|
||||
std::uint32_t cp = (c & 0x1F) << 6 | (static_cast<unsigned char>(s[i + 1]) & 0x3F);
|
||||
if (cp < 0x80)
|
||||
return false; // overlong
|
||||
i += 2;
|
||||
} else if ((c & 0xF0) == 0xE0) {
|
||||
if (!cont(i + 1) || !cont(i + 2))
|
||||
return false;
|
||||
std::uint32_t cp = (c & 0x0F) << 12 |
|
||||
(static_cast<unsigned char>(s[i + 1]) & 0x3F) << 6 |
|
||||
(static_cast<unsigned char>(s[i + 2]) & 0x3F);
|
||||
if (cp < 0x800 || (cp >= 0xD800 && cp <= 0xDFFF))
|
||||
return false;
|
||||
i += 3;
|
||||
} else if ((c & 0xF8) == 0xF0) {
|
||||
if (!cont(i + 1) || !cont(i + 2) || !cont(i + 3))
|
||||
return false;
|
||||
std::uint32_t cp = (c & 0x07) << 18 |
|
||||
(static_cast<unsigned char>(s[i + 1]) & 0x3F) << 12 |
|
||||
(static_cast<unsigned char>(s[i + 2]) & 0x3F) << 6 |
|
||||
(static_cast<unsigned char>(s[i + 3]) & 0x3F);
|
||||
if (cp < 0x10000 || cp > 0x10FFFF)
|
||||
return false;
|
||||
i += 4;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string latin1_to_utf8(std::string_view s) {
|
||||
std::string out;
|
||||
out.reserve(s.size() + s.size() / 2);
|
||||
for (char ch : s)
|
||||
append_utf8(out, static_cast<unsigned char>(ch));
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string to_utf8_best_effort(std::string_view s) {
|
||||
return is_valid_utf8(s) ? std::string(s) : latin1_to_utf8(s);
|
||||
}
|
||||
|
||||
std::string base64_decode(std::string_view in) {
|
||||
auto val = [](char c) -> int {
|
||||
if (c >= 'A' && c <= 'Z')
|
||||
return c - 'A';
|
||||
if (c >= 'a' && c <= 'z')
|
||||
return c - 'a' + 26;
|
||||
if (c >= '0' && c <= '9')
|
||||
return c - '0' + 52;
|
||||
if (c == '+')
|
||||
return 62;
|
||||
if (c == '/')
|
||||
return 63;
|
||||
return -1;
|
||||
};
|
||||
std::string out;
|
||||
out.reserve(in.size() / 4 * 3 + 3);
|
||||
std::uint32_t acc = 0;
|
||||
int bits = 0;
|
||||
for (char c : in) {
|
||||
if (c == '=' || c == '\r' || c == '\n' || c == ' ' || c == '\t')
|
||||
continue;
|
||||
int v = val(c);
|
||||
if (v < 0)
|
||||
continue; // skip stray bytes
|
||||
acc = (acc << 6) | static_cast<std::uint32_t>(v);
|
||||
bits += 6;
|
||||
if (bits >= 8) {
|
||||
bits -= 8;
|
||||
out.push_back(static_cast<char>((acc >> bits) & 0xFF));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string decode_rfc2047(std::string_view in, bool *had_encoded_word) {
|
||||
if (had_encoded_word)
|
||||
*had_encoded_word = false;
|
||||
std::string out;
|
||||
out.reserve(in.size());
|
||||
std::size_t i = 0;
|
||||
const std::size_t n = in.size();
|
||||
|
||||
while (i < n) {
|
||||
auto start = in.find("=?", i);
|
||||
if (start == std::string_view::npos) {
|
||||
out.append(in.substr(i));
|
||||
break;
|
||||
}
|
||||
out.append(in.substr(i, start - i));
|
||||
|
||||
// =?charset?enc?text?=
|
||||
auto q1 = in.find('?', start + 2);
|
||||
if (q1 == std::string_view::npos) {
|
||||
out.append(in.substr(start));
|
||||
break;
|
||||
}
|
||||
auto q2 = in.find('?', q1 + 1);
|
||||
if (q2 == std::string_view::npos || q2 != q1 + 2) {
|
||||
out.append("=?");
|
||||
i = start + 2;
|
||||
continue;
|
||||
}
|
||||
auto end = in.find("?=", q2 + 1);
|
||||
if (end == std::string_view::npos) {
|
||||
out.append(in.substr(start));
|
||||
break;
|
||||
}
|
||||
|
||||
std::string_view charset = in.substr(start + 2, q1 - (start + 2));
|
||||
char enc = in[q1 + 1];
|
||||
std::string_view text = in.substr(q2 + 1, end - (q2 + 1));
|
||||
|
||||
std::string bytes;
|
||||
if (enc == 'B' || enc == 'b') {
|
||||
bytes = base64_decode(text);
|
||||
} else if (enc == 'Q' || enc == 'q') {
|
||||
for (std::size_t k = 0; k < text.size(); ++k) {
|
||||
char c = text[k];
|
||||
if (c == '_') {
|
||||
bytes.push_back(' ');
|
||||
} else if (c == '=' && k + 2 < text.size()) {
|
||||
int hi = hex_val(text[k + 1]);
|
||||
int lo = hex_val(text[k + 2]);
|
||||
if (hi >= 0 && lo >= 0) {
|
||||
bytes.push_back(static_cast<char>((hi << 4) | lo));
|
||||
k += 2;
|
||||
} else {
|
||||
bytes.push_back(c);
|
||||
}
|
||||
} else {
|
||||
bytes.push_back(c);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
out.append(in.substr(start, end + 2 - start)); // unknown encoding: verbatim
|
||||
i = end + 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (is_utf8(charset))
|
||||
out.append(to_utf8_best_effort(bytes));
|
||||
else if (is_latin1(charset))
|
||||
out.append(latin1_to_utf8(bytes));
|
||||
else
|
||||
out.append(to_utf8_best_effort(bytes));
|
||||
|
||||
if (had_encoded_word)
|
||||
*had_encoded_word = true;
|
||||
i = end + 2;
|
||||
|
||||
// RFC 2047: whitespace between adjacent encoded words is elided.
|
||||
std::size_t j = i;
|
||||
while (j < n && (in[j] == ' ' || in[j] == '\t'))
|
||||
++j;
|
||||
if (j < n && in.compare(j, 2, "=?") == 0)
|
||||
i = j;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace vdm::net::detail
|
||||
@@ -0,0 +1,40 @@
|
||||
// vdm/net/text_codec.hpp — internal: small byte/text codecs for header parsing.
|
||||
//
|
||||
// Not a public header. Everything here is pure, allocation-bounded, and total (no throw,
|
||||
// no assert on input): the inputs come off the wire from untrusted servers.
|
||||
|
||||
#ifndef VDM_NET_TEXT_CODEC_HPP
|
||||
#define VDM_NET_TEXT_CODEC_HPP
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace vdm::net::detail {
|
||||
|
||||
// Percent-decode ("%XX"). A stray '%' or a non-hex digit after it is emitted literally.
|
||||
// `plus_as_space` handles application/x-www-form-urlencoded style; off for URL paths.
|
||||
[[nodiscard]] std::string percent_decode(std::string_view in, bool plus_as_space = false);
|
||||
|
||||
// True if `s` is well-formed UTF-8 (no overlong forms, no surrogates, no > U+10FFFF).
|
||||
[[nodiscard]] bool is_valid_utf8(std::string_view s) noexcept;
|
||||
|
||||
// Reinterpret each byte as a Latin-1 (ISO-8859-1) code point and re-encode as UTF-8.
|
||||
[[nodiscard]] std::string latin1_to_utf8(std::string_view s);
|
||||
|
||||
// Decode standard base64 (RFC 4648, '+' '/', optional '=' padding). Whitespace is
|
||||
// skipped. Invalid trailing bits are dropped. Returns the decoded bytes.
|
||||
[[nodiscard]] std::string base64_decode(std::string_view in);
|
||||
|
||||
// Decode RFC 2047 "encoded-word" runs: =?charset?B?..?= / =?charset?Q?..?=. Text outside
|
||||
// encoded words is passed through. Only UTF-8 and ISO-8859-1/Latin-1 charsets are
|
||||
// transcoded; anything else is passed through as-is (best effort). `had_encoded_word`
|
||||
// reports whether at least one well-formed word was found.
|
||||
[[nodiscard]] std::string decode_rfc2047(std::string_view in, bool *had_encoded_word = nullptr);
|
||||
|
||||
// If `s` is valid UTF-8, return it unchanged; otherwise treat it as Latin-1 and transcode.
|
||||
[[nodiscard]] std::string to_utf8_best_effort(std::string_view s);
|
||||
|
||||
} // namespace vdm::net::detail
|
||||
|
||||
#endif // VDM_NET_TEXT_CODEC_HPP
|
||||
@@ -0,0 +1,126 @@
|
||||
// vdm/net/url.cpp
|
||||
|
||||
#include "vdm/net/url.hpp"
|
||||
|
||||
#include <charconv>
|
||||
|
||||
#include "net/text_codec.hpp"
|
||||
|
||||
namespace vdm::net {
|
||||
namespace {
|
||||
|
||||
std::string ascii_lower(std::string_view s) {
|
||||
std::string r(s);
|
||||
for (char &c : r)
|
||||
if (c >= 'A' && c <= 'Z')
|
||||
c = static_cast<char>(c - 'A' + 'a');
|
||||
return r;
|
||||
}
|
||||
|
||||
bool valid_scheme(std::string_view s) {
|
||||
if (s.empty())
|
||||
return false;
|
||||
if (!((s[0] >= 'a' && s[0] <= 'z') || (s[0] >= 'A' && s[0] <= 'Z')))
|
||||
return false;
|
||||
for (char c : s) {
|
||||
bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
|
||||
c == '+' || c == '-' || c == '.';
|
||||
if (!ok)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
SplitUrl split_url(std::string_view url) {
|
||||
SplitUrl out;
|
||||
|
||||
auto scheme_end = url.find("://");
|
||||
if (scheme_end == std::string_view::npos)
|
||||
return out;
|
||||
std::string_view scheme = url.substr(0, scheme_end);
|
||||
if (!valid_scheme(scheme))
|
||||
return out;
|
||||
out.scheme = ascii_lower(scheme);
|
||||
|
||||
std::string_view rest = url.substr(scheme_end + 3);
|
||||
|
||||
// authority ends at the first '/', '?' or '#'
|
||||
std::size_t auth_end = rest.size();
|
||||
for (std::size_t i = 0; i < rest.size(); ++i) {
|
||||
char c = rest[i];
|
||||
if (c == '/' || c == '?' || c == '#') {
|
||||
auth_end = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
std::string_view authority = rest.substr(0, auth_end);
|
||||
std::string_view tail = rest.substr(auth_end);
|
||||
|
||||
if (auto at = authority.rfind('@'); at != std::string_view::npos) {
|
||||
out.userinfo.assign(authority.substr(0, at));
|
||||
authority = authority.substr(at + 1);
|
||||
}
|
||||
|
||||
std::string_view host = authority;
|
||||
std::string_view port;
|
||||
if (!authority.empty() && authority.front() == '[') {
|
||||
auto close = authority.find(']');
|
||||
if (close != std::string_view::npos) {
|
||||
host = authority.substr(1, close - 1); // strip brackets
|
||||
if (close + 1 < authority.size() && authority[close + 1] == ':')
|
||||
port = authority.substr(close + 2);
|
||||
}
|
||||
} else if (auto colon = authority.rfind(':'); colon != std::string_view::npos) {
|
||||
host = authority.substr(0, colon);
|
||||
port = authority.substr(colon + 1);
|
||||
}
|
||||
out.host = ascii_lower(host);
|
||||
|
||||
if (!port.empty()) {
|
||||
unsigned v = 0;
|
||||
auto [p, ec] = std::from_chars(port.data(), port.data() + port.size(), v);
|
||||
(void)p;
|
||||
if (ec == std::errc{} && v > 0 && v <= 65535)
|
||||
out.port = static_cast<std::uint16_t>(v);
|
||||
}
|
||||
|
||||
// tail = path [ '?' query ] [ '#' fragment ]
|
||||
std::string_view path_and_rest = tail;
|
||||
if (auto hash = path_and_rest.find('#'); hash != std::string_view::npos) {
|
||||
out.fragment.assign(path_and_rest.substr(hash + 1));
|
||||
path_and_rest = path_and_rest.substr(0, hash);
|
||||
}
|
||||
if (auto q = path_and_rest.find('?'); q != std::string_view::npos) {
|
||||
out.query.assign(path_and_rest.substr(q + 1));
|
||||
path_and_rest = path_and_rest.substr(0, q);
|
||||
}
|
||||
out.path.assign(path_and_rest);
|
||||
|
||||
out.valid = !out.host.empty() && out.is_http();
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string url_filename(std::string_view url) {
|
||||
SplitUrl u = split_url(url);
|
||||
std::string_view path = u.path;
|
||||
if (path.empty())
|
||||
return {};
|
||||
|
||||
auto slash = path.find_last_of('/');
|
||||
std::string_view seg = (slash == std::string_view::npos) ? path : path.substr(slash + 1);
|
||||
if (seg.empty())
|
||||
return {};
|
||||
|
||||
std::string name = detail::percent_decode(seg);
|
||||
// Guard against a decoded segment that reintroduces a separator or NUL.
|
||||
for (char &c : name)
|
||||
if (c == '/' || c == '\\' || c == '\0')
|
||||
c = '_';
|
||||
if (name == "." || name == "..")
|
||||
return {};
|
||||
return name;
|
||||
}
|
||||
|
||||
} // namespace vdm::net
|
||||
Reference in New Issue
Block a user