diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 5cb5b82..1140360 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(veloxcore STATIC src/net/probe.cpp src/io/sparse_file.cpp src/io/write_buffer.cpp + src/meta/veloxpart.cpp ) add_library(velox::core ALIAS veloxcore) diff --git a/core/include/vdm/meta/veloxpart.hpp b/core/include/vdm/meta/veloxpart.hpp new file mode 100644 index 0000000..2f46d8e --- /dev/null +++ b/core/include/vdm/meta/veloxpart.hpp @@ -0,0 +1,109 @@ +// vdm/meta/veloxpart.hpp — the `.veloxpart.meta` resume sidecar (docs/04 §5). +// +// Written next to the part file so a download survives a daemon crash, a reboot, and a +// database loss. Little-endian, versioned, CRC-32 over the whole record, fdatasync'd at +// segment boundaries. +// +// The reader is written first and fuzzed (AGENT-CORE §5): this file lives in a +// world-writable-ish download directory, so parse_veloxpart() is total on hostile input — +// every malformation is a Result error (meta_corrupt / meta_version_unsupported), never a +// crash, an over-read, or an unbounded allocation. +// +// On-disk layout (all integers little-endian): +// +// magic "VDMP" 4 bytes +// version u16 (this build writes/reads kVersion) +// flags u16 (bit0: sha256_state present) +// total_size u64 (0 = unknown / chunked) +// downloaded u64 (sum of segment.completed; a fast read) +// url_count u32 +// url[0] = original, url[1] = effective, url[2..] = mirrors, each length-prefixed +// etag length-prefixed UTF-8 +// last_modified length-prefixed UTF-8 +// content_type length-prefixed UTF-8 +// segment_count u32 +// per segment: start u64, end u64 (INCLUSIVE), completed u64 +// [flags bit0] sha256_state_len u32, then that many opaque bytes +// crc32 u32 (over every byte before this field) +// +// length-prefixed = u32 length, then that many bytes. +// +// This header compiles standalone. + +#ifndef VDM_META_VELOXPART_HPP +#define VDM_META_VELOXPART_HPP + +#include +#include +#include +#include + +#include "vdm/util/bytes.hpp" +#include "vdm/util/result.hpp" + +namespace vdm::meta { + +inline constexpr std::uint16_t kVersion = 1; +inline constexpr std::uint16_t kFlagHasShaState = 0x0001; + +// Hard caps the reader enforces so a hostile count/length can't drive allocation or work. +inline constexpr std::uint32_t kMaxUrls = 64; +inline constexpr std::uint32_t kMaxSegments = 1024; // contract ceiling is 32; headroom +inline constexpr std::uint32_t kMaxStringLen = 16 * 1024; +inline constexpr std::uint32_t kMaxShaStateLen = 4 * 1024; +inline constexpr std::size_t kMaxImageBytes = 256 * 1024; // a real sidecar is < 4 KiB + +struct SegmentRecord { + std::uint64_t start = 0; + std::uint64_t end = 0; // INCLUSIVE, per contract Segment.endByte / ADR 0010 + std::uint64_t completed = 0; + + [[nodiscard]] std::uint64_t length() const noexcept { + return end >= start ? end - start + 1 : 0; + } + bool operator==(const SegmentRecord &) const = default; +}; + +struct VeloxPart { + std::uint16_t version = kVersion; + std::uint16_t flags = 0; + std::uint64_t total_size = 0; + std::uint64_t downloaded = 0; + std::vector urls; // [0]=original, [1]=effective, [2..]=mirrors + std::string etag; + std::string last_modified; + std::string content_type; + std::vector segments; + std::vector sha256_state; + + [[nodiscard]] std::string_view original_url() const { + return urls.empty() ? std::string_view{} : std::string_view(urls[0]); + } + [[nodiscard]] std::string_view effective_url() const { + return urls.size() < 2 ? original_url() : std::string_view(urls[1]); + } + + bool operator==(const VeloxPart &) const = default; +}; + +// Parse a sidecar image. Every failure is a Result error, never a throw or a crash: +// meta_corrupt — bad magic, truncation, a count/length past a cap or past +// the buffer, trailing bytes, or a CRC mismatch +// meta_version_unsupported — magic OK, CRC OK, but version > kVersion +[[nodiscard]] Result parse_veloxpart(ConstByteSpan image); + +// Serialize. Deterministic: the same VeloxPart always produces the same bytes, so an +// unchanged sidecar is not rewritten. The CRC-32 is appended. +[[nodiscard]] std::vector serialize_veloxpart(const VeloxPart &vp); + +// File helpers — the sidecar path is `.veloxpart.meta`. +[[nodiscard]] Result read_veloxpart_file(std::string_view path); + +// Writes atomically (temp + rename) and, when `fsync`, fdatasync's the file and its +// directory before returning — call at every segment-boundary update (docs/04 §5). +[[nodiscard]] Result write_veloxpart_file(std::string_view path, const VeloxPart &vp, + bool fsync = true); + +} // namespace vdm::meta + +#endif // VDM_META_VELOXPART_HPP diff --git a/core/include/vdm/net/content_disposition.hpp b/core/include/vdm/net/content_disposition.hpp index 7696d15..a84a8f2 100644 --- a/core/include/vdm/net/content_disposition.hpp +++ b/core/include/vdm/net/content_disposition.hpp @@ -21,9 +21,10 @@ struct ContentDisposition { Type type = Type::none; - // Best-effort UTF-8 filename, path components stripped, or empty when the header - // carries none. NOT sanitized for the filesystem — that is rules/ (stage 9); this - // only decodes and de-mojibakes. `..` and control characters may still be present. + // Best-effort UTF-8 filename: path components stripped, control bytes (incl. NUL) and + // edge whitespace removed, or empty when the header carries none. NOT fully sanitized + // for the filesystem — that is rules/ (stage 9). `..`, reserved names, and other + // printable-but-unsafe content may still be present. std::string filename; // The filename came from an RFC 5987 `filename*` ext-value (preferred over a plain diff --git a/core/include/vdm/util/crc32.hpp b/core/include/vdm/util/crc32.hpp new file mode 100644 index 0000000..13a575c --- /dev/null +++ b/core/include/vdm/util/crc32.hpp @@ -0,0 +1,48 @@ +// vdm/util/crc32.hpp — CRC-32 (IEEE 802.3 / zlib polynomial), header-only. +// +// Used to integrity-check the .veloxpart.meta resume sidecar (docs/04 §5). Standard +// reflected CRC-32 with 0xEDB88320, init/xorout 0xFFFFFFFF — byte-compatible with +// zlib's crc32() and `cksum -o3` — so the value is reproducible outside this codebase. +// +// This header compiles standalone. + +#ifndef VDM_UTIL_CRC32_HPP +#define VDM_UTIL_CRC32_HPP + +#include +#include +#include + +#include "vdm/util/bytes.hpp" + +namespace vdm { + +namespace detail { +inline constexpr std::array make_crc32_table() { + std::array t{}; + for (std::uint32_t i = 0; i < 256; ++i) { + std::uint32_t c = i; + for (int k = 0; k < 8; ++k) + c = (c & 1u) ? (0xEDB88320u ^ (c >> 1)) : (c >> 1); + t[i] = c; + } + return t; +} +inline constexpr std::array kCrc32Table = make_crc32_table(); +} // namespace detail + +// Incremental: pass the previous result back as `seed` to continue over split buffers. +[[nodiscard]] inline std::uint32_t crc32_update(std::uint32_t seed, ConstByteSpan data) noexcept { + std::uint32_t c = seed ^ 0xFFFFFFFFu; + for (std::byte b : data) + c = detail::kCrc32Table[(c ^ std::to_integer(b)) & 0xFFu] ^ (c >> 8); + return c ^ 0xFFFFFFFFu; +} + +[[nodiscard]] inline std::uint32_t crc32(ConstByteSpan data) noexcept { + return crc32_update(0u, data); +} + +} // namespace vdm + +#endif // VDM_UTIL_CRC32_HPP diff --git a/core/src/meta/veloxpart.cpp b/core/src/meta/veloxpart.cpp new file mode 100644 index 0000000..0446788 --- /dev/null +++ b/core/src/meta/veloxpart.cpp @@ -0,0 +1,311 @@ +// vdm/meta/veloxpart.cpp +// +// Reader first (AGENT-CORE §5). parse_veloxpart() is the attacker-facing surface; it is +// total on any byte string. + +#include "vdm/meta/veloxpart.hpp" + +#include +#include + +#include +#include +#include +#include + +#include "vdm/util/crc32.hpp" + +namespace vdm::meta { +namespace { + +// magic(4)+ver(2)+flags(2)+total(8)+downloaded(8)+url_count(4) +// +etag_len(4)+lm_len(4)+ct_len(4)+seg_count(4)+crc(4) +constexpr std::size_t kMinImageBytes = 52; +constexpr char kMagic[4] = {'V', 'D', 'M', 'P'}; + +Error errno_to_error(int e) noexcept { + switch (e) { + case ENOSPC: + case EDQUOT: + return Error::disk_full; + case EACCES: + case EPERM: + case EROFS: + return Error::permission_denied; + case ENOENT: + case ENOTDIR: + case EISDIR: + case ENAMETOOLONG: + case ELOOP: + return Error::path_rejected; + default: + return Error::io_error; + } +} + +ErrorInfo sys_error(std::string_view what, int e) { + return ErrorInfo(errno_to_error(e), std::string(what) + ": " + std::strerror(e)); +} + +ErrorInfo corrupt(std::string_view where) { + return ErrorInfo(Error::meta_corrupt, std::string(".veloxpart.meta: ") + std::string(where)); +} + +} // namespace + +// --------------------------------------------------------------------------------------- +// Reader + +Result parse_veloxpart(ConstByteSpan image) { + if (image.size() > kMaxImageBytes) + return corrupt("image exceeds cap"); + if (image.size() < kMinImageBytes) + return corrupt("image shorter than the header"); + + // CRC over everything but the trailing u32 — reject before interpreting any field. + const ConstByteSpan body = image.first(image.size() - 4); + const std::uint32_t want = load_le(image.subspan(image.size() - 4)); + if (crc32(body) != want) + return corrupt("crc32 mismatch"); + + ByteReader r(body); + + if (as_chars(r.raw(4)) != std::string_view(kMagic, 4)) + return corrupt("bad magic"); + + VeloxPart vp; + vp.version = r.u16(); + vp.flags = r.u16(); + if (vp.version > kVersion) + return ErrorInfo(Error::meta_version_unsupported, + ".veloxpart.meta: version " + std::to_string(vp.version) + + " > supported " + std::to_string(kVersion)); + + vp.total_size = r.u64(); + vp.downloaded = r.u64(); + + const std::uint32_t url_count = r.u32(); + if (url_count > kMaxUrls) + return corrupt("url_count past cap"); + if (static_cast(url_count) * 4 > r.remaining()) + return corrupt("url_count"); + vp.urls.reserve(url_count); + for (std::uint32_t i = 0; i < url_count; ++i) { + std::string_view s = r.lp_string(); + if (r.overran() || s.size() > kMaxStringLen) + return corrupt("url"); + vp.urls.emplace_back(s); + } + + auto read_str = [&](std::string &dst, std::string_view what) -> Result { + std::string_view s = r.lp_string(); + if (r.overran() || s.size() > kMaxStringLen) + return corrupt(what); + dst.assign(s); + return ok(); + }; + VDM_TRY(read_str(vp.etag, "etag")); + VDM_TRY(read_str(vp.last_modified, "last_modified")); + VDM_TRY(read_str(vp.content_type, "content_type")); + + const std::uint32_t seg_count = r.u32(); + if (seg_count > kMaxSegments) + return corrupt("segment_count past cap"); + if (static_cast(seg_count) * 24 > r.remaining()) + return corrupt("segment_count"); + vp.segments.reserve(seg_count); + for (std::uint32_t i = 0; i < seg_count; ++i) { + SegmentRecord s; + s.start = r.u64(); + s.end = r.u64(); + s.completed = r.u64(); + if (r.overran()) + return corrupt("segment record"); + if (s.end >= s.start && s.completed > s.end - s.start + 1) + return corrupt("segment.completed exceeds its range"); + vp.segments.push_back(s); + } + + if (vp.flags & kFlagHasShaState) { + const std::uint32_t n = r.u32(); + if (r.overran() || n > kMaxShaStateLen) + return corrupt("sha256_state length"); + ConstByteSpan blob = r.raw(n); + if (r.overran()) + return corrupt("sha256_state body"); + vp.sha256_state.assign(blob.begin(), blob.end()); + } + + if (r.overran()) + return corrupt("truncated"); + if (r.remaining() != 0) + return corrupt("trailing bytes after the record"); + + return vp; +} + +// --------------------------------------------------------------------------------------- +// Writer + +std::vector serialize_veloxpart(const VeloxPart &vp) { + std::vector out; + std::size_t est = kMinImageBytes + vp.segments.size() * 24 + vp.sha256_state.size() + 64; + for (const auto &u : vp.urls) + est += 4 + u.size(); + est += vp.etag.size() + vp.last_modified.size() + vp.content_type.size(); + out.reserve(est); + + auto put_bytes = [&](const void *p, std::size_t n) { + const auto *b = static_cast(p); + out.insert(out.end(), b, b + n); + }; + auto put_u16 = [&](std::uint16_t v) { + std::byte t[2]; + store_le(t, v); + put_bytes(t, 2); + }; + auto put_u32 = [&](std::uint32_t v) { + std::byte t[4]; + store_le(t, v); + put_bytes(t, 4); + }; + auto put_u64 = [&](std::uint64_t v) { + std::byte t[8]; + store_le(t, v); + put_bytes(t, 8); + }; + auto put_str = [&](std::string_view s) { + put_u32(static_cast(s.size())); + put_bytes(s.data(), s.size()); + }; + + // Normalise the sha-state flag to match the payload so a round-trip is exact. + std::uint16_t flags = vp.flags; + if (vp.sha256_state.empty()) + flags &= static_cast(~kFlagHasShaState); + else + flags |= kFlagHasShaState; + + put_bytes(kMagic, 4); + put_u16(vp.version); + put_u16(flags); + put_u64(vp.total_size); + put_u64(vp.downloaded); + put_u32(static_cast(vp.urls.size())); + for (const auto &u : vp.urls) + put_str(u); + put_str(vp.etag); + put_str(vp.last_modified); + put_str(vp.content_type); + put_u32(static_cast(vp.segments.size())); + for (const auto &s : vp.segments) { + put_u64(s.start); + put_u64(s.end); + put_u64(s.completed); + } + if (!vp.sha256_state.empty()) { + put_u32(static_cast(vp.sha256_state.size())); + put_bytes(vp.sha256_state.data(), vp.sha256_state.size()); + } + + put_u32(crc32(ConstByteSpan(out.data(), out.size()))); + return out; +} + +// --------------------------------------------------------------------------------------- +// File helpers + +Result read_veloxpart_file(std::string_view path) { + std::string p(path); + int fd = ::open(p.c_str(), O_RDONLY | O_CLOEXEC); + if (fd < 0) + return sys_error("open " + p, errno); + + std::vector buf; + buf.resize(kMaxImageBytes + 1); + std::size_t total = 0; + for (;;) { + ssize_t n = ::read(fd, buf.data() + total, buf.size() - total); + if (n < 0) { + if (errno == EINTR) + continue; + int e = errno; + ::close(fd); + return sys_error("read " + p, e); + } + if (n == 0) + break; + total += static_cast(n); + if (total > kMaxImageBytes) { + ::close(fd); + return corrupt("sidecar file exceeds cap"); + } + } + ::close(fd); + buf.resize(total); + return parse_veloxpart(ConstByteSpan(buf.data(), buf.size())); +} + +Result write_veloxpart_file(std::string_view path, const VeloxPart &vp, bool fsync) { + std::string p(path); + std::string tmp = p + ".tmp"; + std::vector image = serialize_veloxpart(vp); + + int fd = ::open(tmp.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); + if (fd < 0) + return sys_error("open " + tmp, errno); + + const std::byte *pd = image.data(); + std::size_t remaining = image.size(); + while (remaining > 0) { + ssize_t n = ::write(fd, pd, remaining); + if (n < 0) { + if (errno == EINTR) + continue; + int e = errno; + ::close(fd); + ::unlink(tmp.c_str()); + return sys_error("write " + tmp, e); + } + pd += n; + remaining -= static_cast(n); + } + + if (fsync) { + while (::fdatasync(fd) != 0) { + if (errno == EINTR) + continue; + int e = errno; + ::close(fd); + ::unlink(tmp.c_str()); + return sys_error("fdatasync " + tmp, e); + } + } + if (::close(fd) != 0) { + int e = errno; + ::unlink(tmp.c_str()); + return sys_error("close " + tmp, e); + } + + if (::rename(tmp.c_str(), p.c_str()) != 0) { + int e = errno; + ::unlink(tmp.c_str()); + return sys_error("rename " + tmp + " -> " + p, e); + } + + if (fsync) { + // fsync the directory so the rename itself is durable. + std::string dir = p.substr(0, p.find_last_of('/')); + if (dir.empty() || dir == p) + dir = "."; + int dfd = ::open(dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (dfd >= 0) { + while (::fsync(dfd) != 0 && errno == EINTR) { + } + ::close(dfd); + } + } + return ok(); +} + +} // namespace vdm::meta diff --git a/core/src/net/content_disposition.cpp b/core/src/net/content_disposition.cpp index d86aab6..14ef451 100644 --- a/core/src/net/content_disposition.cpp +++ b/core/src/net/content_disposition.cpp @@ -111,6 +111,20 @@ Params tokenize(std::string_view h) { return out; } +// Drop C0 control bytes and DEL, then trim edge whitespace. NUL and control characters +// are never a legitimate part of a filename and are a classic truncation/spoofing vector, +// so the decode layer strips them even though rules/ (stage 9) owns the authoritative +// sanitize. `..` and other "unsafe but printable" content is left for rules/. +std::string sanitize_leaf(std::string s) { + std::string out; + out.reserve(s.size()); + for (unsigned char c : s) + if (c >= 0x20 && c != 0x7F) + out.push_back(static_cast(c)); + std::string_view v = trim(out); + return std::string(v); +} + std::string strip_path(std::string s) { auto slash = s.find_last_of("/\\"); if (slash != std::string::npos) @@ -270,9 +284,8 @@ ContentDisposition parse_content_disposition(std::string_view header_value) { cd.filename_from_ext = false; } - // Trim ASCII whitespace the decoders may have left at the edges. - std::string_view fv = trim(cd.filename); - cd.filename.assign(fv); + // Drop control bytes (incl. NUL) and edge whitespace the decoders may have produced. + cd.filename = sanitize_leaf(std::move(cd.filename)); return cd; } diff --git a/core/tests/CMakeLists.txt b/core/tests/CMakeLists.txt index e215eab..8ea7ed8 100644 --- a/core/tests/CMakeLists.txt +++ b/core/tests/CMakeLists.txt @@ -27,6 +27,7 @@ vdm_add_test(veloxcore_content_disposition_test net/content_disposition_test.cpp vdm_add_test(veloxcore_url_test net/url_test.cpp) vdm_add_test(veloxcore_sparse_file_test io/sparse_file_test.cpp) vdm_add_test(veloxcore_write_buffer_test io/write_buffer_test.cpp) +vdm_add_test(veloxcore_veloxpart_test meta/veloxpart_test.cpp) set(_testserver ${CMAKE_SOURCE_DIR}/tools/testserver/testserver.py) foreach(net_it http_client probe) diff --git a/core/tests/meta/veloxpart_test.cpp b/core/tests/meta/veloxpart_test.cpp new file mode 100644 index 0000000..4635d42 --- /dev/null +++ b/core/tests/meta/veloxpart_test.cpp @@ -0,0 +1,246 @@ +#include "vdm/meta/veloxpart.hpp" + +#include + +#include +#include +#include +#include +#include + +#include "vdm/util/crc32.hpp" +#include "vtest.hpp" + +using namespace vdm; +using namespace vdm::meta; + +namespace { + +VeloxPart sample_full() { + VeloxPart vp; + vp.version = kVersion; + vp.total_size = 5'000'000'000ull; + vp.downloaded = 1'234'567; + vp.urls = {"https://origin.example/big.iso", "https://cdn.example/big.iso?sig=abc", + "https://mirror1.example/big.iso"}; + vp.etag = "\"deadbeef-1234\""; + vp.last_modified = "Wed, 01 Jan 2025 00:00:00 GMT"; + vp.content_type = "application/octet-stream"; + for (int i = 0; i < 8; ++i) { + SegmentRecord s; + s.start = static_cast(i) * 625'000'000ull; + s.end = s.start + 625'000'000ull - 1; // inclusive + s.completed = (i < 2) ? s.end - s.start + 1 : 100'000ull * (i + 1); + vp.segments.push_back(s); + } + vp.flags = kFlagHasShaState; + vp.sha256_state = {std::byte{1}, std::byte{2}, std::byte{3}, std::byte{0xAA}, std::byte{0xFF}}; + return vp; +} + +VeloxPart sample_minimal() { + VeloxPart vp; + vp.total_size = 0; // chunked / unknown + vp.urls = {"http://x/y"}; + SegmentRecord s; + s.start = 0; + s.end = 0; // 1-byte resource + s.completed = 0; + vp.segments.push_back(s); + return vp; +} + +// Re-CRC a mutated body (everything except the final u32). +std::vector refresh_crc(std::vector image) { + std::uint32_t c = crc32(ConstByteSpan(image.data(), image.size() - 4)); + for (int i = 0; i < 4; ++i) + image[image.size() - 4 + i] = static_cast((c >> (8 * i)) & 0xFF); + return image; +} + +struct TempPath { + std::string path; + TempPath() { + const char *d = std::getenv("TMPDIR"); + path = (d ? d : "/tmp"); + path += "/vdm_vp_test_XXXXXX"; + int fd = ::mkstemp(path.data()); + if (fd >= 0) { + ::close(fd); + ::unlink(path.c_str()); + } + } + ~TempPath() { + ::unlink(path.c_str()); + ::unlink((path + ".tmp").c_str()); + } +}; + +} // namespace + +VT_TEST(crc32_known_vector) { + const char *s = "123456789"; + VT_CHECK_EQ(crc32(ConstByteSpan(reinterpret_cast(s), 9)), 0xCBF43926u); + VT_CHECK_EQ(crc32(ConstByteSpan{}), 0u); +} + +VT_TEST(vp_roundtrip_full) { + VeloxPart in = sample_full(); + auto image = serialize_veloxpart(in); + auto out = parse_veloxpart(ConstByteSpan(image.data(), image.size())); + VT_REQUIRE(out.has_value()); + VT_CHECK(out.value() == in); + VT_CHECK_EQ(out.value().effective_url(), + std::string_view("https://cdn.example/big.iso?sig=abc")); + VT_CHECK_EQ(out.value().segments.size(), 8u); +} + +VT_TEST(vp_roundtrip_minimal) { + VeloxPart in = sample_minimal(); + auto image = serialize_veloxpart(in); + auto out = parse_veloxpart(ConstByteSpan(image.data(), image.size())); + VT_REQUIRE(out.has_value()); + VT_CHECK(out.value() == in); + VT_CHECK(out.value().sha256_state.empty()); +} + +VT_TEST(vp_serialize_is_deterministic) { + VeloxPart in = sample_full(); + auto a = serialize_veloxpart(in); + auto b = serialize_veloxpart(in); + VT_CHECK(a == b); +} + +VT_TEST(vp_file_roundtrip) { + TempPath tp; + VeloxPart in = sample_full(); + VT_REQUIRE(write_veloxpart_file(tp.path, in, /*fsync=*/true).has_value()); + auto out = read_veloxpart_file(tp.path); + VT_REQUIRE(out.has_value()); + VT_CHECK(out.value() == in); + // the temp file must be gone after the atomic rename + VT_CHECK_EQ(::access((tp.path + ".tmp").c_str(), F_OK), -1); +} + +VT_TEST(vp_read_missing_file_errors) { + auto out = read_veloxpart_file("/vdm_no_such_dir_zz/x.veloxpart.meta"); + VT_REQUIRE(!out.has_value()); + VT_CHECK_EQ(out.error().code, Error::path_rejected); +} + +// --- truncation / corruption table ------------------------------------------------- + +VT_TEST(vp_reject_empty_and_tiny) { + VT_CHECK_EQ(parse_veloxpart(ConstByteSpan{}).error().code, Error::meta_corrupt); + std::array three{}; + VT_CHECK_EQ(parse_veloxpart(ConstByteSpan(three.data(), 3)).error().code, Error::meta_corrupt); + std::array almost{}; + VT_CHECK_EQ(parse_veloxpart(ConstByteSpan(almost.data(), almost.size())).error().code, + Error::meta_corrupt); +} + +VT_TEST(vp_reject_bad_magic) { + auto image = serialize_veloxpart(sample_minimal()); + image[1] = std::byte{'X'}; // "VDMP" -> "VXMP" + image = refresh_crc(std::move(image)); // fix CRC so we're testing the magic check + auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size())); + VT_REQUIRE(!r.has_value()); + VT_CHECK_EQ(r.error().code, Error::meta_corrupt); +} + +VT_TEST(vp_reject_crc_mismatch) { + auto image = serialize_veloxpart(sample_minimal()); + image[20] ^= std::byte{0x40}; // flip a payload bit, do NOT refresh the CRC + auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size())); + VT_REQUIRE(!r.has_value()); + VT_CHECK_EQ(r.error().code, Error::meta_corrupt); + + // flipping the CRC field itself is also a mismatch + auto image2 = serialize_veloxpart(sample_minimal()); + image2.back() ^= std::byte{0xFF}; + VT_CHECK_EQ(parse_veloxpart(ConstByteSpan(image2.data(), image2.size())).error().code, + Error::meta_corrupt); +} + +VT_TEST(vp_future_version_is_unsupported) { + auto image = serialize_veloxpart(sample_minimal()); + image[4] = std::byte{99}; // version u16 low byte + image[5] = std::byte{0}; + image = refresh_crc(std::move(image)); + auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size())); + VT_REQUIRE(!r.has_value()); + VT_CHECK_EQ(r.error().code, Error::meta_version_unsupported); +} + +VT_TEST(vp_reject_truncation_at_every_stage) { + auto full = serialize_veloxpart(sample_full()); + // Chop the image at many lengths; each must be meta_corrupt, never a crash. + for (std::size_t len = full.size() - 1; len >= 1; len = (len > 8 ? len - 7 : len - 1)) { + auto r = parse_veloxpart(ConstByteSpan(full.data(), len)); + VT_CHECK(!r.has_value()); + if (r.has_value()) + break; + VT_CHECK_EQ(r.error().code, Error::meta_corrupt); + if (len == 1) + break; + } +} + +VT_TEST(vp_reject_hostile_url_count) { + auto image = serialize_veloxpart(sample_minimal()); + // url_count u32 sits right after magic(4)+ver(2)+flags(2)+total(8)+downloaded(8) = 24 + for (int i = 0; i < 4; ++i) + image[24 + i] = std::byte{0xFF}; + image = refresh_crc(std::move(image)); + auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size())); + VT_REQUIRE(!r.has_value()); + VT_CHECK_EQ(r.error().code, Error::meta_corrupt); +} + +VT_TEST(vp_reject_hostile_lp_string_length) { + // The case AGENT-CORE singles out: a length prefix that reaches past the buffer. + // Build a minimal image, then overwrite url[0]'s length prefix with a huge value. + auto image = serialize_veloxpart(sample_minimal()); + // layout up to url[0] length: magic4 ver2 flags2 total8 downloaded8 url_count4 = 28 + for (int i = 0; i < 4; ++i) + image[28 + i] = std::byte{0xFF}; // url[0] len = 4 GiB - 1 + image = refresh_crc(std::move(image)); + auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size())); + VT_REQUIRE(!r.has_value()); + VT_CHECK_EQ(r.error().code, Error::meta_corrupt); +} + +VT_TEST(vp_reject_hostile_segment_count) { + auto image = serialize_veloxpart(sample_minimal()); + // find the segment_count: it's u32 right before the (24-byte) segment records and the + // trailing crc. minimal sample has 1 segment and no sha state, so: + // segment_count is at size - 4(crc) - 24(one segment) - 4 = size - 32 + std::size_t sc = image.size() - 32; + for (int i = 0; i < 4; ++i) + image[sc + i] = std::byte{0xFF}; + image = refresh_crc(std::move(image)); + auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size())); + VT_REQUIRE(!r.has_value()); + VT_CHECK_EQ(r.error().code, Error::meta_corrupt); +} + +VT_TEST(vp_reject_trailing_bytes) { + auto image = serialize_veloxpart(sample_minimal()); + image.push_back(std::byte{0}); + image.push_back(std::byte{0}); + image = refresh_crc(std::move(image)); // CRC now covers the padding too + auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size())); + VT_REQUIRE(!r.has_value()); + VT_CHECK_EQ(r.error().code, Error::meta_corrupt); +} + +VT_TEST(vp_reject_segment_completed_over_length) { + VeloxPart in = sample_minimal(); + in.segments[0].start = 0; + in.segments[0].end = 99; // length 100 + in.segments[0].completed = 500; // impossible + auto image = serialize_veloxpart(in); + auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size())); + VT_REQUIRE(!r.has_value()); + VT_CHECK_EQ(r.error().code, Error::meta_corrupt); +} diff --git a/core/tests/net/content_disposition_test.cpp b/core/tests/net/content_disposition_test.cpp index 61057db..d48dfa1 100644 --- a/core/tests/net/content_disposition_test.cpp +++ b/core/tests/net/content_disposition_test.cpp @@ -135,6 +135,23 @@ VT_TEST(cd_bad_percent_escapes_in_ext_value) { VT_CHECK(cd.type == Type::attachment); } +VT_TEST(cd_strips_control_bytes_and_nul) { + // A mangled ext-value that decodes to bytes with embedded NULs (fuzz-found). + std::string h1("attachment; filename*=x''%e2%82%a"); + h1.push_back('\0'); + h1.push_back('\0'); + h1 += "ff.pdf"; + auto cd = parse_content_disposition(h1); + for (unsigned char c : cd.filename) + VT_CHECK(c >= 0x20 && c != 0x7F); + // a plain filename with a tab / newline / SOH loses them + std::string h2("attachment; filename=\"a\tb\nc"); + h2.push_back('\x01'); + h2 += ".txt\""; + auto cd2 = parse_content_disposition(h2); + VT_CHECK_EQ(cd2.filename, std::string("abc.txt")); +} + VT_TEST(cd_case_insensitive_keys_and_type) { auto cd = parse_content_disposition(R"(ATTACHMENT; FileName="x.txt")"); VT_CHECK(cd.type == Type::attachment); diff --git a/tools/fuzz/CMakeLists.txt b/tools/fuzz/CMakeLists.txt index 5f9f9d7..241f151 100644 --- a/tools/fuzz/CMakeLists.txt +++ b/tools/fuzz/CMakeLists.txt @@ -19,24 +19,38 @@ endif() set(_core ${CMAKE_SOURCE_DIR}/core) set(_fuzz_flags -g -O1 -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer) -function(vdm_add_fuzzer name) +# corpus_dir is the seed set (checked in, small, hand-named). It is passed read-only: +# the CTest smoke run uses -runs so libFuzzer exits, and a real campaign is +# `bin/ corpus//` by hand. +function(vdm_add_fuzzer name corpus_dir) add_executable(${name} ${ARGN}) target_include_directories(${name} PRIVATE ${_core}/include ${_core}/src) target_compile_features(${name} PRIVATE cxx_std_23) target_compile_options(${name} PRIVATE ${_fuzz_flags}) target_link_options(${name} PRIVATE ${_fuzz_flags}) + + if(VELOX_BUILD_TESTS) + # Regression tripwire only: replay the checked-in seeds once (-runs=0, no + # mutation, no corpus writes) so a parser change that breaks a known-good or + # known-hostile input fails CI in a fraction of a second. The 1M-exec bar + # (AGENT-CORE M1 DoD) is a separate campaign job: `bin/ corpus//`. + add_test(NAME ${name}_smoke + COMMAND ${name} -runs=0 ${CMAKE_CURRENT_SOURCE_DIR}/${corpus_dir}) + set_tests_properties(${name}_smoke PROPERTIES LABELS "fuzz" TIMEOUT 60) + endif() endfunction() -vdm_add_fuzzer(fuzz_content_disposition +vdm_add_fuzzer(fuzz_content_disposition corpus/content_disposition content_disposition_fuzz.cpp ${_core}/src/net/content_disposition.cpp ${_core}/src/net/text_codec.cpp) -vdm_add_fuzzer(fuzz_url +vdm_add_fuzzer(fuzz_url corpus/url url_fuzz.cpp ${_core}/src/net/url.cpp ${_core}/src/net/text_codec.cpp) -# Seed corpora live next to the harnesses. -file(GLOB _cd_seeds ${CMAKE_CURRENT_SOURCE_DIR}/corpus/content_disposition/*) -file(GLOB _url_seeds ${CMAKE_CURRENT_SOURCE_DIR}/corpus/url/*) +vdm_add_fuzzer(fuzz_veloxpart corpus/veloxpart + veloxpart_fuzz.cpp + ${_core}/src/meta/veloxpart.cpp + ${_core}/src/util/error.cpp) diff --git a/tools/fuzz/content_disposition_fuzz.cpp b/tools/fuzz/content_disposition_fuzz.cpp index 65e2cc3..01ba914 100644 --- a/tools/fuzz/content_disposition_fuzz.cpp +++ b/tools/fuzz/content_disposition_fuzz.cpp @@ -14,9 +14,10 @@ extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size std::string_view header(reinterpret_cast(data), size); auto cd = vdm::net::parse_content_disposition(header); - // Light invariants: a returned filename never contains a path separator or NUL. - for (char c : cd.filename) - if (c == '/' || c == '\\' || c == '\0') + // Invariant: a returned filename has no path separator and no control byte (incl. + // NUL / DEL). `..` and other printable-but-unsafe content is rules/'s to handle. + for (unsigned char c : cd.filename) + if (c == '/' || c == '\\' || c < 0x20 || c == 0x7F) __builtin_trap(); return 0; diff --git a/tools/fuzz/corpus/veloxpart/magic_only b/tools/fuzz/corpus/veloxpart/magic_only new file mode 100644 index 0000000..1633b1b --- /dev/null +++ b/tools/fuzz/corpus/veloxpart/magic_only @@ -0,0 +1 @@ +VDMP \ No newline at end of file diff --git a/tools/fuzz/corpus/veloxpart/valid b/tools/fuzz/corpus/veloxpart/valid new file mode 100644 index 0000000..e393b5d Binary files /dev/null and b/tools/fuzz/corpus/veloxpart/valid differ diff --git a/tools/fuzz/corpus/veloxpart/zeros40 b/tools/fuzz/corpus/veloxpart/zeros40 new file mode 100644 index 0000000..6459c29 Binary files /dev/null and b/tools/fuzz/corpus/veloxpart/zeros40 differ diff --git a/tools/fuzz/veloxpart_fuzz.cpp b/tools/fuzz/veloxpart_fuzz.cpp new file mode 100644 index 0000000..29b84a7 --- /dev/null +++ b/tools/fuzz/veloxpart_fuzz.cpp @@ -0,0 +1,52 @@ +// Fuzz target for the .veloxpart.meta reader — the file AGENT-CORE §5 calls +// attacker-adjacent (it lives in a world-writable-ish download directory). The reader +// must be total: no crash, no over-read, no unbounded allocation, on ANY byte string. +// +// clang++ -std=c++23 -fsanitize=fuzzer,address,undefined ... (see CMakeLists.txt) +// ./fuzz_veloxpart -max_len=8192 corpus/veloxpart/ + +#include +#include +#include + +#include "vdm/meta/veloxpart.hpp" +#include "vdm/util/bytes.hpp" +#include "vdm/util/crc32.hpp" + +using vdm::ConstByteSpan; +using vdm::meta::parse_veloxpart; +using vdm::meta::serialize_veloxpart; +using vdm::meta::VeloxPart; + +namespace { + +void check_roundtrip_stable(const VeloxPart &vp) { + // A value the reader accepted must serialize and re-parse to an equal value — + // otherwise the reader is accepting something the writer can't reproduce. + auto image = serialize_veloxpart(vp); + auto again = parse_veloxpart(ConstByteSpan(image.data(), image.size())); + if (!again.has_value() || !(again.value() == vp)) + __builtin_trap(); +} + +} // namespace + +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { + ConstByteSpan raw(reinterpret_cast(data), size); + + // 1. Raw bytes straight in — most inputs die at the magic or CRC check. + if (auto r = parse_veloxpart(raw); r.has_value()) + check_roundtrip_stable(r.value()); + + // 2. Same bytes with a valid CRC-32 appended, so the field parser is actually + // reached and the ByteReader bounds checks (and load_le's precondition behind + // them) get exercised on structurally-plausible-but-hostile input. + std::vector with_crc(raw.begin(), raw.end()); + std::uint32_t c = vdm::crc32(raw); + for (int i = 0; i < 4; ++i) + with_crc.push_back(static_cast((c >> (8 * i)) & 0xFF)); + if (auto r = parse_veloxpart(ConstByteSpan(with_crc.data(), with_crc.size())); r.has_value()) + check_roundtrip_stable(r.value()); + + return 0; +}