core: net/probe + Content-Disposition parser + URL splitter (stage 3)
net/content_disposition — total parser for the mojibake-prone header: RFC 6266 filename (quoted/token), RFC 5987 filename* ext-values (charset'lang'pct-encoded, incl. RFC 2231 continuations), legacy RFC 2047 encoded-words (=?UTF-8?B?..?= / ?Q?), and raw Latin-1 bytes; prefers filename* over filename; strips path components AFTER decoding (a base64 payload can hold '/'). 22-case test table. net/text_codec (internal) — percent-decode, UTF-8 validation, Latin-1-> UTF-8, base64, RFC 2047 — shared by the CD parser and the URL splitter. net/url — a small total URL splitter (scheme/userinfo/host/port/path/ query/fragment, http(s) validity) and url_filename() for the last path segment; used for the filename fallback. net/probe — HEAD then a ranged GET bytes=0-0 that PROVES resumability (206 + matching Content-Range + a validator), rather than trusting Accept-Ranges which servers lie about; the ranged GET is also the HEAD- refused (403/405/501) fallback. 401/407 -> success result with requires_auth, not an error. Runs on its own pool (max_concurrent, default 4) outside the segment budget per ADR 0011 §5. suggest_filename() does the resolution order (explicit -> disposition -> URL -> download.bin) with a light strip; rules/ (stage 9) owns the authoritative sanitize. tools/fuzz — libFuzzer targets for the CD parser and the URL splitter, compiling the parser sources directly so they're fully instrumented; self-guards on VELOX_BUILD_FUZZ + Clang (the top-level CMake adds every tools/* unconditionally). Seed corpora included. Fixed on the way: a p -> Transfer -> State -> cbs -> p reference cycle in Prober that leaked every probe (drop the stored Transfer; the worker keeps State alive). Tests green under ASan/UBSan and TSan. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01HPPSGhiArbvQgwC2DNiURS
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,262 @@
|
||||
// vdm/net/text_codec.cpp
|
||||
|
||||
#include "net/text_codec.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <cctype>
|
||||
|
||||
namespace vdm::net::detail {
|
||||
namespace {
|
||||
|
||||
int hex_val(char c) noexcept {
|
||||
if (c >= '0' && c <= '9')
|
||||
return c - '0';
|
||||
if (c >= 'a' && c <= 'f')
|
||||
return c - 'a' + 10;
|
||||
if (c >= 'A' && c <= 'F')
|
||||
return c - 'A' + 10;
|
||||
return -1;
|
||||
}
|
||||
|
||||
void append_utf8(std::string &out, std::uint32_t cp) {
|
||||
if (cp <= 0x7F) {
|
||||
out.push_back(static_cast<char>(cp));
|
||||
} else if (cp <= 0x7FF) {
|
||||
out.push_back(static_cast<char>(0xC0 | (cp >> 6)));
|
||||
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
|
||||
} else if (cp <= 0xFFFF) {
|
||||
out.push_back(static_cast<char>(0xE0 | (cp >> 12)));
|
||||
out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
|
||||
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
|
||||
} else {
|
||||
out.push_back(static_cast<char>(0xF0 | (cp >> 18)));
|
||||
out.push_back(static_cast<char>(0x80 | ((cp >> 12) & 0x3F)));
|
||||
out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
|
||||
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
|
||||
}
|
||||
}
|
||||
|
||||
bool charset_is(std::string_view cs, std::string_view want) {
|
||||
if (cs.size() != want.size())
|
||||
return false;
|
||||
for (std::size_t i = 0; i < cs.size(); ++i) {
|
||||
char a = cs[i], b = want[i];
|
||||
if (a >= 'A' && a <= 'Z')
|
||||
a = static_cast<char>(a - 'A' + 'a');
|
||||
if (b >= 'A' && b <= 'Z')
|
||||
b = static_cast<char>(b - 'A' + 'a');
|
||||
if (a != b)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool is_utf8(std::string_view cs) {
|
||||
return charset_is(cs, "utf-8") || charset_is(cs, "utf8");
|
||||
}
|
||||
bool is_latin1(std::string_view cs) {
|
||||
return charset_is(cs, "iso-8859-1") || charset_is(cs, "latin1") ||
|
||||
charset_is(cs, "iso8859-1") || charset_is(cs, "windows-1252");
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string percent_decode(std::string_view in, bool plus_as_space) {
|
||||
std::string out;
|
||||
out.reserve(in.size());
|
||||
for (std::size_t i = 0; i < in.size(); ++i) {
|
||||
char c = in[i];
|
||||
if (c == '%' && i + 2 < in.size()) {
|
||||
int hi = hex_val(in[i + 1]);
|
||||
int lo = hex_val(in[i + 2]);
|
||||
if (hi >= 0 && lo >= 0) {
|
||||
out.push_back(static_cast<char>((hi << 4) | lo));
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (c == '+' && plus_as_space) {
|
||||
out.push_back(' ');
|
||||
continue;
|
||||
}
|
||||
out.push_back(c);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
bool is_valid_utf8(std::string_view s) noexcept {
|
||||
std::size_t i = 0;
|
||||
const std::size_t n = s.size();
|
||||
auto cont = [&](std::size_t k) {
|
||||
return k < n && (static_cast<unsigned char>(s[k]) & 0xC0) == 0x80;
|
||||
};
|
||||
while (i < n) {
|
||||
unsigned char c = static_cast<unsigned char>(s[i]);
|
||||
if (c < 0x80) {
|
||||
++i;
|
||||
} else if ((c & 0xE0) == 0xC0) {
|
||||
if (!cont(i + 1))
|
||||
return false;
|
||||
std::uint32_t cp = (c & 0x1F) << 6 | (static_cast<unsigned char>(s[i + 1]) & 0x3F);
|
||||
if (cp < 0x80)
|
||||
return false; // overlong
|
||||
i += 2;
|
||||
} else if ((c & 0xF0) == 0xE0) {
|
||||
if (!cont(i + 1) || !cont(i + 2))
|
||||
return false;
|
||||
std::uint32_t cp = (c & 0x0F) << 12 |
|
||||
(static_cast<unsigned char>(s[i + 1]) & 0x3F) << 6 |
|
||||
(static_cast<unsigned char>(s[i + 2]) & 0x3F);
|
||||
if (cp < 0x800 || (cp >= 0xD800 && cp <= 0xDFFF))
|
||||
return false;
|
||||
i += 3;
|
||||
} else if ((c & 0xF8) == 0xF0) {
|
||||
if (!cont(i + 1) || !cont(i + 2) || !cont(i + 3))
|
||||
return false;
|
||||
std::uint32_t cp = (c & 0x07) << 18 |
|
||||
(static_cast<unsigned char>(s[i + 1]) & 0x3F) << 12 |
|
||||
(static_cast<unsigned char>(s[i + 2]) & 0x3F) << 6 |
|
||||
(static_cast<unsigned char>(s[i + 3]) & 0x3F);
|
||||
if (cp < 0x10000 || cp > 0x10FFFF)
|
||||
return false;
|
||||
i += 4;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string latin1_to_utf8(std::string_view s) {
|
||||
std::string out;
|
||||
out.reserve(s.size() + s.size() / 2);
|
||||
for (char ch : s)
|
||||
append_utf8(out, static_cast<unsigned char>(ch));
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string to_utf8_best_effort(std::string_view s) {
|
||||
return is_valid_utf8(s) ? std::string(s) : latin1_to_utf8(s);
|
||||
}
|
||||
|
||||
std::string base64_decode(std::string_view in) {
|
||||
auto val = [](char c) -> int {
|
||||
if (c >= 'A' && c <= 'Z')
|
||||
return c - 'A';
|
||||
if (c >= 'a' && c <= 'z')
|
||||
return c - 'a' + 26;
|
||||
if (c >= '0' && c <= '9')
|
||||
return c - '0' + 52;
|
||||
if (c == '+')
|
||||
return 62;
|
||||
if (c == '/')
|
||||
return 63;
|
||||
return -1;
|
||||
};
|
||||
std::string out;
|
||||
out.reserve(in.size() / 4 * 3 + 3);
|
||||
std::uint32_t acc = 0;
|
||||
int bits = 0;
|
||||
for (char c : in) {
|
||||
if (c == '=' || c == '\r' || c == '\n' || c == ' ' || c == '\t')
|
||||
continue;
|
||||
int v = val(c);
|
||||
if (v < 0)
|
||||
continue; // skip stray bytes
|
||||
acc = (acc << 6) | static_cast<std::uint32_t>(v);
|
||||
bits += 6;
|
||||
if (bits >= 8) {
|
||||
bits -= 8;
|
||||
out.push_back(static_cast<char>((acc >> bits) & 0xFF));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string decode_rfc2047(std::string_view in, bool *had_encoded_word) {
|
||||
if (had_encoded_word)
|
||||
*had_encoded_word = false;
|
||||
std::string out;
|
||||
out.reserve(in.size());
|
||||
std::size_t i = 0;
|
||||
const std::size_t n = in.size();
|
||||
|
||||
while (i < n) {
|
||||
auto start = in.find("=?", i);
|
||||
if (start == std::string_view::npos) {
|
||||
out.append(in.substr(i));
|
||||
break;
|
||||
}
|
||||
out.append(in.substr(i, start - i));
|
||||
|
||||
// =?charset?enc?text?=
|
||||
auto q1 = in.find('?', start + 2);
|
||||
if (q1 == std::string_view::npos) {
|
||||
out.append(in.substr(start));
|
||||
break;
|
||||
}
|
||||
auto q2 = in.find('?', q1 + 1);
|
||||
if (q2 == std::string_view::npos || q2 != q1 + 2) {
|
||||
out.append("=?");
|
||||
i = start + 2;
|
||||
continue;
|
||||
}
|
||||
auto end = in.find("?=", q2 + 1);
|
||||
if (end == std::string_view::npos) {
|
||||
out.append(in.substr(start));
|
||||
break;
|
||||
}
|
||||
|
||||
std::string_view charset = in.substr(start + 2, q1 - (start + 2));
|
||||
char enc = in[q1 + 1];
|
||||
std::string_view text = in.substr(q2 + 1, end - (q2 + 1));
|
||||
|
||||
std::string bytes;
|
||||
if (enc == 'B' || enc == 'b') {
|
||||
bytes = base64_decode(text);
|
||||
} else if (enc == 'Q' || enc == 'q') {
|
||||
for (std::size_t k = 0; k < text.size(); ++k) {
|
||||
char c = text[k];
|
||||
if (c == '_') {
|
||||
bytes.push_back(' ');
|
||||
} else if (c == '=' && k + 2 < text.size()) {
|
||||
int hi = hex_val(text[k + 1]);
|
||||
int lo = hex_val(text[k + 2]);
|
||||
if (hi >= 0 && lo >= 0) {
|
||||
bytes.push_back(static_cast<char>((hi << 4) | lo));
|
||||
k += 2;
|
||||
} else {
|
||||
bytes.push_back(c);
|
||||
}
|
||||
} else {
|
||||
bytes.push_back(c);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
out.append(in.substr(start, end + 2 - start)); // unknown encoding: verbatim
|
||||
i = end + 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (is_utf8(charset))
|
||||
out.append(to_utf8_best_effort(bytes));
|
||||
else if (is_latin1(charset))
|
||||
out.append(latin1_to_utf8(bytes));
|
||||
else
|
||||
out.append(to_utf8_best_effort(bytes));
|
||||
|
||||
if (had_encoded_word)
|
||||
*had_encoded_word = true;
|
||||
i = end + 2;
|
||||
|
||||
// RFC 2047: whitespace between adjacent encoded words is elided.
|
||||
std::size_t j = i;
|
||||
while (j < n && (in[j] == ' ' || in[j] == '\t'))
|
||||
++j;
|
||||
if (j < n && in.compare(j, 2, "=?") == 0)
|
||||
i = j;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace vdm::net::detail
|
||||
Reference in New Issue
Block a user