core: meta/veloxpart — resume sidecar, reader first + fuzzed (stage 5)
util/crc32.hpp — header-only CRC-32 (zlib polynomial, reflected), used to integrity-check the sidecar. meta/veloxpart — the <name>.veloxpart.meta resume file (docs/04 §5). Little-endian, versioned, CRC-32 over the whole record. Layout: magic, version, flags, total_size, downloaded, url set (original/effective/ mirrors), etag/last-modified/content-type, segment records (start, end INCLUSIVE, completed), optional sha256 streaming-hash blob. parse_veloxpart() is the attacker-facing surface (the file sits in a world-writable-ish download dir) and is total on any byte string: CRC checked before any field is interpreted; magic, a version it understands, every count and length bounded by a hard cap AND checked against the remaining buffer; ByteReader latches on overrun; trailing bytes rejected. Every malformation is meta_corrupt / meta_version_unsupported, never a crash or an unbounded allocation. serialize_veloxpart() is deterministic (unchanged sidecar isn't rewritten). File helpers write atomically (temp + rename) and fdatasync the file and its directory. Tests: crc32 known vector; full + minimal round-trips; deterministic serialize; file round-trip; and a truncation/corruption table — bad magic, CRC mismatch (payload and CRC-field flips), future version, truncation at every stage, hostile url_count / segment_count / lp_string length (the case the brief singles out), trailing bytes, impossible segment.completed. tools/fuzz/fuzz_veloxpart — feeds raw bytes and bytes-with-valid-CRC (so the field parser and ByteReader bounds checks are actually reached), and round-trip-stability-checks anything accepted. Ran 1.1M execs clean under ASan+UBSan+libFuzzer (clang++-21); fuzz_content_disposition and fuzz_url likewise re-run to 1.1M. tools/fuzz gains a -runs=0 seed-replay CTest smoke per target (regression tripwire; the campaign stays manual). Fuzz-found and fixed: parse_content_disposition could emit a filename containing NUL / control bytes from a mangled filename* ext-value — strip_path only removed path separators. Now sanitize_leaf() also drops C0 controls and DEL (rules/ still owns the authoritative sanitize; `..` and printable-unsafe content pass through as before). Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01HPPSGhiArbvQgwC2DNiURS
This commit is contained in:
@@ -19,6 +19,7 @@ add_library(veloxcore STATIC
|
||||
src/net/probe.cpp
|
||||
src/io/sparse_file.cpp
|
||||
src/io/write_buffer.cpp
|
||||
src/meta/veloxpart.cpp
|
||||
)
|
||||
add_library(velox::core ALIAS veloxcore)
|
||||
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
// vdm/meta/veloxpart.hpp — the `<name>.veloxpart.meta` resume sidecar (docs/04 §5).
|
||||
//
|
||||
// Written next to the part file so a download survives a daemon crash, a reboot, and a
|
||||
// database loss. Little-endian, versioned, CRC-32 over the whole record, fdatasync'd at
|
||||
// segment boundaries.
|
||||
//
|
||||
// The reader is written first and fuzzed (AGENT-CORE §5): this file lives in a
|
||||
// world-writable-ish download directory, so parse_veloxpart() is total on hostile input —
|
||||
// every malformation is a Result error (meta_corrupt / meta_version_unsupported), never a
|
||||
// crash, an over-read, or an unbounded allocation.
|
||||
//
|
||||
// On-disk layout (all integers little-endian):
|
||||
//
|
||||
// magic "VDMP" 4 bytes
|
||||
// version u16 (this build writes/reads kVersion)
|
||||
// flags u16 (bit0: sha256_state present)
|
||||
// total_size u64 (0 = unknown / chunked)
|
||||
// downloaded u64 (sum of segment.completed; a fast read)
|
||||
// url_count u32
|
||||
// url[0] = original, url[1] = effective, url[2..] = mirrors, each length-prefixed
|
||||
// etag length-prefixed UTF-8
|
||||
// last_modified length-prefixed UTF-8
|
||||
// content_type length-prefixed UTF-8
|
||||
// segment_count u32
|
||||
// per segment: start u64, end u64 (INCLUSIVE), completed u64
|
||||
// [flags bit0] sha256_state_len u32, then that many opaque bytes
|
||||
// crc32 u32 (over every byte before this field)
|
||||
//
|
||||
// length-prefixed = u32 length, then that many bytes.
|
||||
//
|
||||
// This header compiles standalone.
|
||||
|
||||
#ifndef VDM_META_VELOXPART_HPP
|
||||
#define VDM_META_VELOXPART_HPP
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "vdm/util/bytes.hpp"
|
||||
#include "vdm/util/result.hpp"
|
||||
|
||||
namespace vdm::meta {
|
||||
|
||||
inline constexpr std::uint16_t kVersion = 1;
|
||||
inline constexpr std::uint16_t kFlagHasShaState = 0x0001;
|
||||
|
||||
// Hard caps the reader enforces so a hostile count/length can't drive allocation or work.
|
||||
inline constexpr std::uint32_t kMaxUrls = 64;
|
||||
inline constexpr std::uint32_t kMaxSegments = 1024; // contract ceiling is 32; headroom
|
||||
inline constexpr std::uint32_t kMaxStringLen = 16 * 1024;
|
||||
inline constexpr std::uint32_t kMaxShaStateLen = 4 * 1024;
|
||||
inline constexpr std::size_t kMaxImageBytes = 256 * 1024; // a real sidecar is < 4 KiB
|
||||
|
||||
struct SegmentRecord {
|
||||
std::uint64_t start = 0;
|
||||
std::uint64_t end = 0; // INCLUSIVE, per contract Segment.endByte / ADR 0010
|
||||
std::uint64_t completed = 0;
|
||||
|
||||
[[nodiscard]] std::uint64_t length() const noexcept {
|
||||
return end >= start ? end - start + 1 : 0;
|
||||
}
|
||||
bool operator==(const SegmentRecord &) const = default;
|
||||
};
|
||||
|
||||
struct VeloxPart {
|
||||
std::uint16_t version = kVersion;
|
||||
std::uint16_t flags = 0;
|
||||
std::uint64_t total_size = 0;
|
||||
std::uint64_t downloaded = 0;
|
||||
std::vector<std::string> urls; // [0]=original, [1]=effective, [2..]=mirrors
|
||||
std::string etag;
|
||||
std::string last_modified;
|
||||
std::string content_type;
|
||||
std::vector<SegmentRecord> segments;
|
||||
std::vector<std::byte> sha256_state;
|
||||
|
||||
[[nodiscard]] std::string_view original_url() const {
|
||||
return urls.empty() ? std::string_view{} : std::string_view(urls[0]);
|
||||
}
|
||||
[[nodiscard]] std::string_view effective_url() const {
|
||||
return urls.size() < 2 ? original_url() : std::string_view(urls[1]);
|
||||
}
|
||||
|
||||
bool operator==(const VeloxPart &) const = default;
|
||||
};
|
||||
|
||||
// Parse a sidecar image. Every failure is a Result error, never a throw or a crash:
|
||||
// meta_corrupt — bad magic, truncation, a count/length past a cap or past
|
||||
// the buffer, trailing bytes, or a CRC mismatch
|
||||
// meta_version_unsupported — magic OK, CRC OK, but version > kVersion
|
||||
[[nodiscard]] Result<VeloxPart> parse_veloxpart(ConstByteSpan image);
|
||||
|
||||
// Serialize. Deterministic: the same VeloxPart always produces the same bytes, so an
|
||||
// unchanged sidecar is not rewritten. The CRC-32 is appended.
|
||||
[[nodiscard]] std::vector<std::byte> serialize_veloxpart(const VeloxPart &vp);
|
||||
|
||||
// File helpers — the sidecar path is `<part file>.veloxpart.meta`.
|
||||
[[nodiscard]] Result<VeloxPart> read_veloxpart_file(std::string_view path);
|
||||
|
||||
// Writes atomically (temp + rename) and, when `fsync`, fdatasync's the file and its
|
||||
// directory before returning — call at every segment-boundary update (docs/04 §5).
|
||||
[[nodiscard]] Result<void> write_veloxpart_file(std::string_view path, const VeloxPart &vp,
|
||||
bool fsync = true);
|
||||
|
||||
} // namespace vdm::meta
|
||||
|
||||
#endif // VDM_META_VELOXPART_HPP
|
||||
@@ -21,9 +21,10 @@ struct ContentDisposition {
|
||||
|
||||
Type type = Type::none;
|
||||
|
||||
// Best-effort UTF-8 filename, path components stripped, or empty when the header
|
||||
// carries none. NOT sanitized for the filesystem — that is rules/ (stage 9); this
|
||||
// only decodes and de-mojibakes. `..` and control characters may still be present.
|
||||
// Best-effort UTF-8 filename: path components stripped, control bytes (incl. NUL) and
|
||||
// edge whitespace removed, or empty when the header carries none. NOT fully sanitized
|
||||
// for the filesystem — that is rules/ (stage 9). `..`, reserved names, and other
|
||||
// printable-but-unsafe content may still be present.
|
||||
std::string filename;
|
||||
|
||||
// The filename came from an RFC 5987 `filename*` ext-value (preferred over a plain
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
// vdm/util/crc32.hpp — CRC-32 (IEEE 802.3 / zlib polynomial), header-only.
|
||||
//
|
||||
// Used to integrity-check the .veloxpart.meta resume sidecar (docs/04 §5). Standard
|
||||
// reflected CRC-32 with 0xEDB88320, init/xorout 0xFFFFFFFF — byte-compatible with
|
||||
// zlib's crc32() and `cksum -o3` — so the value is reproducible outside this codebase.
|
||||
//
|
||||
// This header compiles standalone.
|
||||
|
||||
#ifndef VDM_UTIL_CRC32_HPP
|
||||
#define VDM_UTIL_CRC32_HPP
|
||||
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
#include "vdm/util/bytes.hpp"
|
||||
|
||||
namespace vdm {
|
||||
|
||||
namespace detail {
|
||||
inline constexpr std::array<std::uint32_t, 256> make_crc32_table() {
|
||||
std::array<std::uint32_t, 256> t{};
|
||||
for (std::uint32_t i = 0; i < 256; ++i) {
|
||||
std::uint32_t c = i;
|
||||
for (int k = 0; k < 8; ++k)
|
||||
c = (c & 1u) ? (0xEDB88320u ^ (c >> 1)) : (c >> 1);
|
||||
t[i] = c;
|
||||
}
|
||||
return t;
|
||||
}
|
||||
inline constexpr std::array<std::uint32_t, 256> kCrc32Table = make_crc32_table();
|
||||
} // namespace detail
|
||||
|
||||
// Incremental: pass the previous result back as `seed` to continue over split buffers.
|
||||
[[nodiscard]] inline std::uint32_t crc32_update(std::uint32_t seed, ConstByteSpan data) noexcept {
|
||||
std::uint32_t c = seed ^ 0xFFFFFFFFu;
|
||||
for (std::byte b : data)
|
||||
c = detail::kCrc32Table[(c ^ std::to_integer<std::uint8_t>(b)) & 0xFFu] ^ (c >> 8);
|
||||
return c ^ 0xFFFFFFFFu;
|
||||
}
|
||||
|
||||
[[nodiscard]] inline std::uint32_t crc32(ConstByteSpan data) noexcept {
|
||||
return crc32_update(0u, data);
|
||||
}
|
||||
|
||||
} // namespace vdm
|
||||
|
||||
#endif // VDM_UTIL_CRC32_HPP
|
||||
@@ -0,0 +1,311 @@
|
||||
// vdm/meta/veloxpart.cpp
|
||||
//
|
||||
// Reader first (AGENT-CORE §5). parse_veloxpart() is the attacker-facing surface; it is
|
||||
// total on any byte string.
|
||||
|
||||
#include "vdm/meta/veloxpart.hpp"
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cerrno>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include "vdm/util/crc32.hpp"
|
||||
|
||||
namespace vdm::meta {
|
||||
namespace {
|
||||
|
||||
// magic(4)+ver(2)+flags(2)+total(8)+downloaded(8)+url_count(4)
|
||||
// +etag_len(4)+lm_len(4)+ct_len(4)+seg_count(4)+crc(4)
|
||||
constexpr std::size_t kMinImageBytes = 52;
|
||||
constexpr char kMagic[4] = {'V', 'D', 'M', 'P'};
|
||||
|
||||
Error errno_to_error(int e) noexcept {
|
||||
switch (e) {
|
||||
case ENOSPC:
|
||||
case EDQUOT:
|
||||
return Error::disk_full;
|
||||
case EACCES:
|
||||
case EPERM:
|
||||
case EROFS:
|
||||
return Error::permission_denied;
|
||||
case ENOENT:
|
||||
case ENOTDIR:
|
||||
case EISDIR:
|
||||
case ENAMETOOLONG:
|
||||
case ELOOP:
|
||||
return Error::path_rejected;
|
||||
default:
|
||||
return Error::io_error;
|
||||
}
|
||||
}
|
||||
|
||||
ErrorInfo sys_error(std::string_view what, int e) {
|
||||
return ErrorInfo(errno_to_error(e), std::string(what) + ": " + std::strerror(e));
|
||||
}
|
||||
|
||||
ErrorInfo corrupt(std::string_view where) {
|
||||
return ErrorInfo(Error::meta_corrupt, std::string(".veloxpart.meta: ") + std::string(where));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// ---------------------------------------------------------------------------------------
|
||||
// Reader
|
||||
|
||||
Result<VeloxPart> parse_veloxpart(ConstByteSpan image) {
|
||||
if (image.size() > kMaxImageBytes)
|
||||
return corrupt("image exceeds cap");
|
||||
if (image.size() < kMinImageBytes)
|
||||
return corrupt("image shorter than the header");
|
||||
|
||||
// CRC over everything but the trailing u32 — reject before interpreting any field.
|
||||
const ConstByteSpan body = image.first(image.size() - 4);
|
||||
const std::uint32_t want = load_le<std::uint32_t>(image.subspan(image.size() - 4));
|
||||
if (crc32(body) != want)
|
||||
return corrupt("crc32 mismatch");
|
||||
|
||||
ByteReader r(body);
|
||||
|
||||
if (as_chars(r.raw(4)) != std::string_view(kMagic, 4))
|
||||
return corrupt("bad magic");
|
||||
|
||||
VeloxPart vp;
|
||||
vp.version = r.u16();
|
||||
vp.flags = r.u16();
|
||||
if (vp.version > kVersion)
|
||||
return ErrorInfo(Error::meta_version_unsupported,
|
||||
".veloxpart.meta: version " + std::to_string(vp.version) +
|
||||
" > supported " + std::to_string(kVersion));
|
||||
|
||||
vp.total_size = r.u64();
|
||||
vp.downloaded = r.u64();
|
||||
|
||||
const std::uint32_t url_count = r.u32();
|
||||
if (url_count > kMaxUrls)
|
||||
return corrupt("url_count past cap");
|
||||
if (static_cast<std::uint64_t>(url_count) * 4 > r.remaining())
|
||||
return corrupt("url_count");
|
||||
vp.urls.reserve(url_count);
|
||||
for (std::uint32_t i = 0; i < url_count; ++i) {
|
||||
std::string_view s = r.lp_string();
|
||||
if (r.overran() || s.size() > kMaxStringLen)
|
||||
return corrupt("url");
|
||||
vp.urls.emplace_back(s);
|
||||
}
|
||||
|
||||
auto read_str = [&](std::string &dst, std::string_view what) -> Result<void> {
|
||||
std::string_view s = r.lp_string();
|
||||
if (r.overran() || s.size() > kMaxStringLen)
|
||||
return corrupt(what);
|
||||
dst.assign(s);
|
||||
return ok();
|
||||
};
|
||||
VDM_TRY(read_str(vp.etag, "etag"));
|
||||
VDM_TRY(read_str(vp.last_modified, "last_modified"));
|
||||
VDM_TRY(read_str(vp.content_type, "content_type"));
|
||||
|
||||
const std::uint32_t seg_count = r.u32();
|
||||
if (seg_count > kMaxSegments)
|
||||
return corrupt("segment_count past cap");
|
||||
if (static_cast<std::uint64_t>(seg_count) * 24 > r.remaining())
|
||||
return corrupt("segment_count");
|
||||
vp.segments.reserve(seg_count);
|
||||
for (std::uint32_t i = 0; i < seg_count; ++i) {
|
||||
SegmentRecord s;
|
||||
s.start = r.u64();
|
||||
s.end = r.u64();
|
||||
s.completed = r.u64();
|
||||
if (r.overran())
|
||||
return corrupt("segment record");
|
||||
if (s.end >= s.start && s.completed > s.end - s.start + 1)
|
||||
return corrupt("segment.completed exceeds its range");
|
||||
vp.segments.push_back(s);
|
||||
}
|
||||
|
||||
if (vp.flags & kFlagHasShaState) {
|
||||
const std::uint32_t n = r.u32();
|
||||
if (r.overran() || n > kMaxShaStateLen)
|
||||
return corrupt("sha256_state length");
|
||||
ConstByteSpan blob = r.raw(n);
|
||||
if (r.overran())
|
||||
return corrupt("sha256_state body");
|
||||
vp.sha256_state.assign(blob.begin(), blob.end());
|
||||
}
|
||||
|
||||
if (r.overran())
|
||||
return corrupt("truncated");
|
||||
if (r.remaining() != 0)
|
||||
return corrupt("trailing bytes after the record");
|
||||
|
||||
return vp;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------
|
||||
// Writer
|
||||
|
||||
std::vector<std::byte> serialize_veloxpart(const VeloxPart &vp) {
|
||||
std::vector<std::byte> out;
|
||||
std::size_t est = kMinImageBytes + vp.segments.size() * 24 + vp.sha256_state.size() + 64;
|
||||
for (const auto &u : vp.urls)
|
||||
est += 4 + u.size();
|
||||
est += vp.etag.size() + vp.last_modified.size() + vp.content_type.size();
|
||||
out.reserve(est);
|
||||
|
||||
auto put_bytes = [&](const void *p, std::size_t n) {
|
||||
const auto *b = static_cast<const std::byte *>(p);
|
||||
out.insert(out.end(), b, b + n);
|
||||
};
|
||||
auto put_u16 = [&](std::uint16_t v) {
|
||||
std::byte t[2];
|
||||
store_le<std::uint16_t>(t, v);
|
||||
put_bytes(t, 2);
|
||||
};
|
||||
auto put_u32 = [&](std::uint32_t v) {
|
||||
std::byte t[4];
|
||||
store_le<std::uint32_t>(t, v);
|
||||
put_bytes(t, 4);
|
||||
};
|
||||
auto put_u64 = [&](std::uint64_t v) {
|
||||
std::byte t[8];
|
||||
store_le<std::uint64_t>(t, v);
|
||||
put_bytes(t, 8);
|
||||
};
|
||||
auto put_str = [&](std::string_view s) {
|
||||
put_u32(static_cast<std::uint32_t>(s.size()));
|
||||
put_bytes(s.data(), s.size());
|
||||
};
|
||||
|
||||
// Normalise the sha-state flag to match the payload so a round-trip is exact.
|
||||
std::uint16_t flags = vp.flags;
|
||||
if (vp.sha256_state.empty())
|
||||
flags &= static_cast<std::uint16_t>(~kFlagHasShaState);
|
||||
else
|
||||
flags |= kFlagHasShaState;
|
||||
|
||||
put_bytes(kMagic, 4);
|
||||
put_u16(vp.version);
|
||||
put_u16(flags);
|
||||
put_u64(vp.total_size);
|
||||
put_u64(vp.downloaded);
|
||||
put_u32(static_cast<std::uint32_t>(vp.urls.size()));
|
||||
for (const auto &u : vp.urls)
|
||||
put_str(u);
|
||||
put_str(vp.etag);
|
||||
put_str(vp.last_modified);
|
||||
put_str(vp.content_type);
|
||||
put_u32(static_cast<std::uint32_t>(vp.segments.size()));
|
||||
for (const auto &s : vp.segments) {
|
||||
put_u64(s.start);
|
||||
put_u64(s.end);
|
||||
put_u64(s.completed);
|
||||
}
|
||||
if (!vp.sha256_state.empty()) {
|
||||
put_u32(static_cast<std::uint32_t>(vp.sha256_state.size()));
|
||||
put_bytes(vp.sha256_state.data(), vp.sha256_state.size());
|
||||
}
|
||||
|
||||
put_u32(crc32(ConstByteSpan(out.data(), out.size())));
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------
|
||||
// File helpers
|
||||
|
||||
Result<VeloxPart> read_veloxpart_file(std::string_view path) {
|
||||
std::string p(path);
|
||||
int fd = ::open(p.c_str(), O_RDONLY | O_CLOEXEC);
|
||||
if (fd < 0)
|
||||
return sys_error("open " + p, errno);
|
||||
|
||||
std::vector<std::byte> buf;
|
||||
buf.resize(kMaxImageBytes + 1);
|
||||
std::size_t total = 0;
|
||||
for (;;) {
|
||||
ssize_t n = ::read(fd, buf.data() + total, buf.size() - total);
|
||||
if (n < 0) {
|
||||
if (errno == EINTR)
|
||||
continue;
|
||||
int e = errno;
|
||||
::close(fd);
|
||||
return sys_error("read " + p, e);
|
||||
}
|
||||
if (n == 0)
|
||||
break;
|
||||
total += static_cast<std::size_t>(n);
|
||||
if (total > kMaxImageBytes) {
|
||||
::close(fd);
|
||||
return corrupt("sidecar file exceeds cap");
|
||||
}
|
||||
}
|
||||
::close(fd);
|
||||
buf.resize(total);
|
||||
return parse_veloxpart(ConstByteSpan(buf.data(), buf.size()));
|
||||
}
|
||||
|
||||
Result<void> write_veloxpart_file(std::string_view path, const VeloxPart &vp, bool fsync) {
|
||||
std::string p(path);
|
||||
std::string tmp = p + ".tmp";
|
||||
std::vector<std::byte> image = serialize_veloxpart(vp);
|
||||
|
||||
int fd = ::open(tmp.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
|
||||
if (fd < 0)
|
||||
return sys_error("open " + tmp, errno);
|
||||
|
||||
const std::byte *pd = image.data();
|
||||
std::size_t remaining = image.size();
|
||||
while (remaining > 0) {
|
||||
ssize_t n = ::write(fd, pd, remaining);
|
||||
if (n < 0) {
|
||||
if (errno == EINTR)
|
||||
continue;
|
||||
int e = errno;
|
||||
::close(fd);
|
||||
::unlink(tmp.c_str());
|
||||
return sys_error("write " + tmp, e);
|
||||
}
|
||||
pd += n;
|
||||
remaining -= static_cast<std::size_t>(n);
|
||||
}
|
||||
|
||||
if (fsync) {
|
||||
while (::fdatasync(fd) != 0) {
|
||||
if (errno == EINTR)
|
||||
continue;
|
||||
int e = errno;
|
||||
::close(fd);
|
||||
::unlink(tmp.c_str());
|
||||
return sys_error("fdatasync " + tmp, e);
|
||||
}
|
||||
}
|
||||
if (::close(fd) != 0) {
|
||||
int e = errno;
|
||||
::unlink(tmp.c_str());
|
||||
return sys_error("close " + tmp, e);
|
||||
}
|
||||
|
||||
if (::rename(tmp.c_str(), p.c_str()) != 0) {
|
||||
int e = errno;
|
||||
::unlink(tmp.c_str());
|
||||
return sys_error("rename " + tmp + " -> " + p, e);
|
||||
}
|
||||
|
||||
if (fsync) {
|
||||
// fsync the directory so the rename itself is durable.
|
||||
std::string dir = p.substr(0, p.find_last_of('/'));
|
||||
if (dir.empty() || dir == p)
|
||||
dir = ".";
|
||||
int dfd = ::open(dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
|
||||
if (dfd >= 0) {
|
||||
while (::fsync(dfd) != 0 && errno == EINTR) {
|
||||
}
|
||||
::close(dfd);
|
||||
}
|
||||
}
|
||||
return ok();
|
||||
}
|
||||
|
||||
} // namespace vdm::meta
|
||||
@@ -111,6 +111,20 @@ Params tokenize(std::string_view h) {
|
||||
return out;
|
||||
}
|
||||
|
||||
// Drop C0 control bytes and DEL, then trim edge whitespace. NUL and control characters
|
||||
// are never a legitimate part of a filename and are a classic truncation/spoofing vector,
|
||||
// so the decode layer strips them even though rules/ (stage 9) owns the authoritative
|
||||
// sanitize. `..` and other "unsafe but printable" content is left for rules/.
|
||||
std::string sanitize_leaf(std::string s) {
|
||||
std::string out;
|
||||
out.reserve(s.size());
|
||||
for (unsigned char c : s)
|
||||
if (c >= 0x20 && c != 0x7F)
|
||||
out.push_back(static_cast<char>(c));
|
||||
std::string_view v = trim(out);
|
||||
return std::string(v);
|
||||
}
|
||||
|
||||
std::string strip_path(std::string s) {
|
||||
auto slash = s.find_last_of("/\\");
|
||||
if (slash != std::string::npos)
|
||||
@@ -270,9 +284,8 @@ ContentDisposition parse_content_disposition(std::string_view header_value) {
|
||||
cd.filename_from_ext = false;
|
||||
}
|
||||
|
||||
// Trim ASCII whitespace the decoders may have left at the edges.
|
||||
std::string_view fv = trim(cd.filename);
|
||||
cd.filename.assign(fv);
|
||||
// Drop control bytes (incl. NUL) and edge whitespace the decoders may have produced.
|
||||
cd.filename = sanitize_leaf(std::move(cd.filename));
|
||||
return cd;
|
||||
}
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ vdm_add_test(veloxcore_content_disposition_test net/content_disposition_test.cpp
|
||||
vdm_add_test(veloxcore_url_test net/url_test.cpp)
|
||||
vdm_add_test(veloxcore_sparse_file_test io/sparse_file_test.cpp)
|
||||
vdm_add_test(veloxcore_write_buffer_test io/write_buffer_test.cpp)
|
||||
vdm_add_test(veloxcore_veloxpart_test meta/veloxpart_test.cpp)
|
||||
|
||||
set(_testserver ${CMAKE_SOURCE_DIR}/tools/testserver/testserver.py)
|
||||
foreach(net_it http_client probe)
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
#include "vdm/meta/veloxpart.hpp"
|
||||
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "vdm/util/crc32.hpp"
|
||||
#include "vtest.hpp"
|
||||
|
||||
using namespace vdm;
|
||||
using namespace vdm::meta;
|
||||
|
||||
namespace {
|
||||
|
||||
VeloxPart sample_full() {
|
||||
VeloxPart vp;
|
||||
vp.version = kVersion;
|
||||
vp.total_size = 5'000'000'000ull;
|
||||
vp.downloaded = 1'234'567;
|
||||
vp.urls = {"https://origin.example/big.iso", "https://cdn.example/big.iso?sig=abc",
|
||||
"https://mirror1.example/big.iso"};
|
||||
vp.etag = "\"deadbeef-1234\"";
|
||||
vp.last_modified = "Wed, 01 Jan 2025 00:00:00 GMT";
|
||||
vp.content_type = "application/octet-stream";
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
SegmentRecord s;
|
||||
s.start = static_cast<std::uint64_t>(i) * 625'000'000ull;
|
||||
s.end = s.start + 625'000'000ull - 1; // inclusive
|
||||
s.completed = (i < 2) ? s.end - s.start + 1 : 100'000ull * (i + 1);
|
||||
vp.segments.push_back(s);
|
||||
}
|
||||
vp.flags = kFlagHasShaState;
|
||||
vp.sha256_state = {std::byte{1}, std::byte{2}, std::byte{3}, std::byte{0xAA}, std::byte{0xFF}};
|
||||
return vp;
|
||||
}
|
||||
|
||||
VeloxPart sample_minimal() {
|
||||
VeloxPart vp;
|
||||
vp.total_size = 0; // chunked / unknown
|
||||
vp.urls = {"http://x/y"};
|
||||
SegmentRecord s;
|
||||
s.start = 0;
|
||||
s.end = 0; // 1-byte resource
|
||||
s.completed = 0;
|
||||
vp.segments.push_back(s);
|
||||
return vp;
|
||||
}
|
||||
|
||||
// Re-CRC a mutated body (everything except the final u32).
|
||||
std::vector<std::byte> refresh_crc(std::vector<std::byte> image) {
|
||||
std::uint32_t c = crc32(ConstByteSpan(image.data(), image.size() - 4));
|
||||
for (int i = 0; i < 4; ++i)
|
||||
image[image.size() - 4 + i] = static_cast<std::byte>((c >> (8 * i)) & 0xFF);
|
||||
return image;
|
||||
}
|
||||
|
||||
struct TempPath {
|
||||
std::string path;
|
||||
TempPath() {
|
||||
const char *d = std::getenv("TMPDIR");
|
||||
path = (d ? d : "/tmp");
|
||||
path += "/vdm_vp_test_XXXXXX";
|
||||
int fd = ::mkstemp(path.data());
|
||||
if (fd >= 0) {
|
||||
::close(fd);
|
||||
::unlink(path.c_str());
|
||||
}
|
||||
}
|
||||
~TempPath() {
|
||||
::unlink(path.c_str());
|
||||
::unlink((path + ".tmp").c_str());
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
VT_TEST(crc32_known_vector) {
|
||||
const char *s = "123456789";
|
||||
VT_CHECK_EQ(crc32(ConstByteSpan(reinterpret_cast<const std::byte *>(s), 9)), 0xCBF43926u);
|
||||
VT_CHECK_EQ(crc32(ConstByteSpan{}), 0u);
|
||||
}
|
||||
|
||||
VT_TEST(vp_roundtrip_full) {
|
||||
VeloxPart in = sample_full();
|
||||
auto image = serialize_veloxpart(in);
|
||||
auto out = parse_veloxpart(ConstByteSpan(image.data(), image.size()));
|
||||
VT_REQUIRE(out.has_value());
|
||||
VT_CHECK(out.value() == in);
|
||||
VT_CHECK_EQ(out.value().effective_url(),
|
||||
std::string_view("https://cdn.example/big.iso?sig=abc"));
|
||||
VT_CHECK_EQ(out.value().segments.size(), 8u);
|
||||
}
|
||||
|
||||
VT_TEST(vp_roundtrip_minimal) {
|
||||
VeloxPart in = sample_minimal();
|
||||
auto image = serialize_veloxpart(in);
|
||||
auto out = parse_veloxpart(ConstByteSpan(image.data(), image.size()));
|
||||
VT_REQUIRE(out.has_value());
|
||||
VT_CHECK(out.value() == in);
|
||||
VT_CHECK(out.value().sha256_state.empty());
|
||||
}
|
||||
|
||||
VT_TEST(vp_serialize_is_deterministic) {
|
||||
VeloxPart in = sample_full();
|
||||
auto a = serialize_veloxpart(in);
|
||||
auto b = serialize_veloxpart(in);
|
||||
VT_CHECK(a == b);
|
||||
}
|
||||
|
||||
VT_TEST(vp_file_roundtrip) {
|
||||
TempPath tp;
|
||||
VeloxPart in = sample_full();
|
||||
VT_REQUIRE(write_veloxpart_file(tp.path, in, /*fsync=*/true).has_value());
|
||||
auto out = read_veloxpart_file(tp.path);
|
||||
VT_REQUIRE(out.has_value());
|
||||
VT_CHECK(out.value() == in);
|
||||
// the temp file must be gone after the atomic rename
|
||||
VT_CHECK_EQ(::access((tp.path + ".tmp").c_str(), F_OK), -1);
|
||||
}
|
||||
|
||||
VT_TEST(vp_read_missing_file_errors) {
|
||||
auto out = read_veloxpart_file("/vdm_no_such_dir_zz/x.veloxpart.meta");
|
||||
VT_REQUIRE(!out.has_value());
|
||||
VT_CHECK_EQ(out.error().code, Error::path_rejected);
|
||||
}
|
||||
|
||||
// --- truncation / corruption table -------------------------------------------------
|
||||
|
||||
VT_TEST(vp_reject_empty_and_tiny) {
|
||||
VT_CHECK_EQ(parse_veloxpart(ConstByteSpan{}).error().code, Error::meta_corrupt);
|
||||
std::array<std::byte, 3> three{};
|
||||
VT_CHECK_EQ(parse_veloxpart(ConstByteSpan(three.data(), 3)).error().code, Error::meta_corrupt);
|
||||
std::array<std::byte, 51> almost{};
|
||||
VT_CHECK_EQ(parse_veloxpart(ConstByteSpan(almost.data(), almost.size())).error().code,
|
||||
Error::meta_corrupt);
|
||||
}
|
||||
|
||||
VT_TEST(vp_reject_bad_magic) {
|
||||
auto image = serialize_veloxpart(sample_minimal());
|
||||
image[1] = std::byte{'X'}; // "VDMP" -> "VXMP"
|
||||
image = refresh_crc(std::move(image)); // fix CRC so we're testing the magic check
|
||||
auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size()));
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::meta_corrupt);
|
||||
}
|
||||
|
||||
VT_TEST(vp_reject_crc_mismatch) {
|
||||
auto image = serialize_veloxpart(sample_minimal());
|
||||
image[20] ^= std::byte{0x40}; // flip a payload bit, do NOT refresh the CRC
|
||||
auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size()));
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::meta_corrupt);
|
||||
|
||||
// flipping the CRC field itself is also a mismatch
|
||||
auto image2 = serialize_veloxpart(sample_minimal());
|
||||
image2.back() ^= std::byte{0xFF};
|
||||
VT_CHECK_EQ(parse_veloxpart(ConstByteSpan(image2.data(), image2.size())).error().code,
|
||||
Error::meta_corrupt);
|
||||
}
|
||||
|
||||
VT_TEST(vp_future_version_is_unsupported) {
|
||||
auto image = serialize_veloxpart(sample_minimal());
|
||||
image[4] = std::byte{99}; // version u16 low byte
|
||||
image[5] = std::byte{0};
|
||||
image = refresh_crc(std::move(image));
|
||||
auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size()));
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::meta_version_unsupported);
|
||||
}
|
||||
|
||||
VT_TEST(vp_reject_truncation_at_every_stage) {
|
||||
auto full = serialize_veloxpart(sample_full());
|
||||
// Chop the image at many lengths; each must be meta_corrupt, never a crash.
|
||||
for (std::size_t len = full.size() - 1; len >= 1; len = (len > 8 ? len - 7 : len - 1)) {
|
||||
auto r = parse_veloxpart(ConstByteSpan(full.data(), len));
|
||||
VT_CHECK(!r.has_value());
|
||||
if (r.has_value())
|
||||
break;
|
||||
VT_CHECK_EQ(r.error().code, Error::meta_corrupt);
|
||||
if (len == 1)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
VT_TEST(vp_reject_hostile_url_count) {
|
||||
auto image = serialize_veloxpart(sample_minimal());
|
||||
// url_count u32 sits right after magic(4)+ver(2)+flags(2)+total(8)+downloaded(8) = 24
|
||||
for (int i = 0; i < 4; ++i)
|
||||
image[24 + i] = std::byte{0xFF};
|
||||
image = refresh_crc(std::move(image));
|
||||
auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size()));
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::meta_corrupt);
|
||||
}
|
||||
|
||||
VT_TEST(vp_reject_hostile_lp_string_length) {
|
||||
// The case AGENT-CORE singles out: a length prefix that reaches past the buffer.
|
||||
// Build a minimal image, then overwrite url[0]'s length prefix with a huge value.
|
||||
auto image = serialize_veloxpart(sample_minimal());
|
||||
// layout up to url[0] length: magic4 ver2 flags2 total8 downloaded8 url_count4 = 28
|
||||
for (int i = 0; i < 4; ++i)
|
||||
image[28 + i] = std::byte{0xFF}; // url[0] len = 4 GiB - 1
|
||||
image = refresh_crc(std::move(image));
|
||||
auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size()));
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::meta_corrupt);
|
||||
}
|
||||
|
||||
VT_TEST(vp_reject_hostile_segment_count) {
|
||||
auto image = serialize_veloxpart(sample_minimal());
|
||||
// find the segment_count: it's u32 right before the (24-byte) segment records and the
|
||||
// trailing crc. minimal sample has 1 segment and no sha state, so:
|
||||
// segment_count is at size - 4(crc) - 24(one segment) - 4 = size - 32
|
||||
std::size_t sc = image.size() - 32;
|
||||
for (int i = 0; i < 4; ++i)
|
||||
image[sc + i] = std::byte{0xFF};
|
||||
image = refresh_crc(std::move(image));
|
||||
auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size()));
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::meta_corrupt);
|
||||
}
|
||||
|
||||
VT_TEST(vp_reject_trailing_bytes) {
|
||||
auto image = serialize_veloxpart(sample_minimal());
|
||||
image.push_back(std::byte{0});
|
||||
image.push_back(std::byte{0});
|
||||
image = refresh_crc(std::move(image)); // CRC now covers the padding too
|
||||
auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size()));
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::meta_corrupt);
|
||||
}
|
||||
|
||||
VT_TEST(vp_reject_segment_completed_over_length) {
|
||||
VeloxPart in = sample_minimal();
|
||||
in.segments[0].start = 0;
|
||||
in.segments[0].end = 99; // length 100
|
||||
in.segments[0].completed = 500; // impossible
|
||||
auto image = serialize_veloxpart(in);
|
||||
auto r = parse_veloxpart(ConstByteSpan(image.data(), image.size()));
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::meta_corrupt);
|
||||
}
|
||||
@@ -135,6 +135,23 @@ VT_TEST(cd_bad_percent_escapes_in_ext_value) {
|
||||
VT_CHECK(cd.type == Type::attachment);
|
||||
}
|
||||
|
||||
VT_TEST(cd_strips_control_bytes_and_nul) {
|
||||
// A mangled ext-value that decodes to bytes with embedded NULs (fuzz-found).
|
||||
std::string h1("attachment; filename*=x''%e2%82%a");
|
||||
h1.push_back('\0');
|
||||
h1.push_back('\0');
|
||||
h1 += "ff.pdf";
|
||||
auto cd = parse_content_disposition(h1);
|
||||
for (unsigned char c : cd.filename)
|
||||
VT_CHECK(c >= 0x20 && c != 0x7F);
|
||||
// a plain filename with a tab / newline / SOH loses them
|
||||
std::string h2("attachment; filename=\"a\tb\nc");
|
||||
h2.push_back('\x01');
|
||||
h2 += ".txt\"";
|
||||
auto cd2 = parse_content_disposition(h2);
|
||||
VT_CHECK_EQ(cd2.filename, std::string("abc.txt"));
|
||||
}
|
||||
|
||||
VT_TEST(cd_case_insensitive_keys_and_type) {
|
||||
auto cd = parse_content_disposition(R"(ATTACHMENT; FileName="x.txt")");
|
||||
VT_CHECK(cd.type == Type::attachment);
|
||||
|
||||
Reference in New Issue
Block a user