io/sparse_file — the single O_WRONLY output file (docs/04 §4). open()
posix_fallocate's the full size (falls back to ftruncate on
EOPNOTSUPP/ENOSYS, reported via preallocated()); write_at() pwrites at an
absolute offset, looping short writes and retrying EINTR; sync() is
fdatasync (timer/pause only); advise_dontneed() is
posix_fadvise(DONTNEED); resize() trims a preallocated tail or sizes a
chunked download. errno -> vdm::Error (ENOSPC->disk_full,
EACCES->permission_denied, ENOENT/ENOTDIR/...->path_rejected). No lock on
write_at — POSIX makes each pwrite atomic for a regular file, so N
segment threads writing disjoint ranges is safe (tested, TSan-clean).
io/write_buffer — per-segment accumulate-and-flush buffer, preallocated
at construction; append() only memcpys (no allocation on the write-
callback hot path, docs/04 §8 — asserted by a global-new counter in the
test). Flushes on fill via a caller-supplied FlushFn; a chunk >= capacity
arriving on an empty buffer writes straight through. On a flush error
next_offset() stays at the last durable position. Single-threaded; the
disk-writer-thread handoff is stage 8.
Also: vtest.hpp VT_CHECK_EQ/NE now copy operands (auto, not auto&&) — an
assertion must not outlive a temporary the expression returned a
reference into (ASan caught this on Result<void>{}.error().code).
Co-Authored-By: Claude Sonnet 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01HPPSGhiArbvQgwC2DNiURS
182 lines
5.2 KiB
C++
182 lines
5.2 KiB
C++
#include "vdm/io/sparse_file.hpp"
|
|
|
|
#include <fcntl.h>
|
|
#include <unistd.h>
|
|
|
|
#include <array>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <string>
|
|
#include <thread>
|
|
#include <vector>
|
|
|
|
#include "vtest.hpp"
|
|
|
|
using vdm::Error;
|
|
using vdm::io::SparseFile;
|
|
|
|
namespace {
|
|
|
|
struct TempPath {
|
|
std::string path;
|
|
TempPath() {
|
|
const char *dir = std::getenv("TMPDIR");
|
|
path = (dir ? dir : "/tmp");
|
|
path += "/vdm_sparse_test_XXXXXX";
|
|
int fd = ::mkstemp(path.data());
|
|
if (fd >= 0) {
|
|
::close(fd);
|
|
::unlink(path.c_str()); // we only wanted a unique name
|
|
}
|
|
}
|
|
~TempPath() { ::unlink(path.c_str()); }
|
|
};
|
|
|
|
std::vector<std::byte> read_all(const std::string &path) {
|
|
int fd = ::open(path.c_str(), O_RDONLY);
|
|
if (fd < 0)
|
|
return {};
|
|
std::vector<std::byte> out;
|
|
std::array<std::byte, 4096> buf{};
|
|
for (;;) {
|
|
ssize_t n = ::read(fd, buf.data(), buf.size());
|
|
if (n <= 0)
|
|
break;
|
|
out.insert(out.end(), buf.begin(), buf.begin() + n);
|
|
}
|
|
::close(fd);
|
|
return out;
|
|
}
|
|
|
|
std::uint64_t file_size(const std::string &path) {
|
|
int fd = ::open(path.c_str(), O_RDONLY);
|
|
if (fd < 0)
|
|
return 0;
|
|
off_t end = ::lseek(fd, 0, SEEK_END);
|
|
::close(fd);
|
|
return end < 0 ? 0 : static_cast<std::uint64_t>(end);
|
|
}
|
|
|
|
vdm::ConstByteSpan bytes(const char *s) {
|
|
return {reinterpret_cast<const std::byte *>(s), std::strlen(s)};
|
|
}
|
|
|
|
} // namespace
|
|
|
|
VT_TEST(sparse_open_preallocates_full_size) {
|
|
TempPath tp;
|
|
SparseFile f;
|
|
auto r = f.open(tp.path, {.total_size = 1 << 20});
|
|
VT_REQUIRE(r.has_value());
|
|
VT_CHECK(f.is_open());
|
|
VT_CHECK_EQ(file_size(tp.path), 1u << 20);
|
|
// /tmp is usually a real fs; if it's tmpfs, preallocated() is false and that's fine.
|
|
VT_CHECK(f.close().has_value());
|
|
}
|
|
|
|
VT_TEST(sparse_write_at_absolute_offsets) {
|
|
TempPath tp;
|
|
SparseFile f;
|
|
VT_REQUIRE(f.open(tp.path, {.total_size = 64}).has_value());
|
|
|
|
VT_CHECK(f.write_at(10, bytes("hello")).has_value());
|
|
VT_CHECK(f.write_at(40, bytes("world")).has_value());
|
|
VT_CHECK(f.sync().has_value());
|
|
|
|
auto data = read_all(tp.path);
|
|
VT_REQUIRE(data.size() == 64);
|
|
VT_CHECK_EQ(std::memcmp(data.data() + 10, "hello", 5), 0);
|
|
VT_CHECK_EQ(std::memcmp(data.data() + 40, "world", 5), 0);
|
|
f.close().value();
|
|
}
|
|
|
|
VT_TEST(sparse_write_past_end_grows_file) {
|
|
TempPath tp;
|
|
SparseFile f;
|
|
VT_REQUIRE(f.open(tp.path, {.total_size = 16}).has_value());
|
|
VT_CHECK(f.write_at(1000, bytes("tail")).has_value());
|
|
VT_CHECK_EQ(file_size(tp.path), 1004u);
|
|
f.close().value();
|
|
}
|
|
|
|
VT_TEST(sparse_resize_trims) {
|
|
TempPath tp;
|
|
SparseFile f;
|
|
VT_REQUIRE(f.open(tp.path, {.total_size = 4096}).has_value());
|
|
VT_CHECK(f.resize(100).has_value());
|
|
VT_CHECK_EQ(file_size(tp.path), 100u);
|
|
f.close().value();
|
|
}
|
|
|
|
VT_TEST(sparse_open_bad_path_is_path_rejected) {
|
|
SparseFile f;
|
|
auto r = f.open("/vdm_no_such_dir_xyz/file.part", {.total_size = 10});
|
|
VT_REQUIRE(!r.has_value());
|
|
VT_CHECK_EQ(r.error().code, Error::path_rejected);
|
|
VT_CHECK(!f.is_open());
|
|
}
|
|
|
|
VT_TEST(sparse_ops_on_closed_file_error) {
|
|
SparseFile f;
|
|
VT_CHECK_EQ(f.write_at(0, bytes("x")).error().code, Error::internal);
|
|
VT_CHECK_EQ(f.sync().error().code, Error::internal);
|
|
VT_CHECK(f.close().has_value()); // close on a closed file is ok
|
|
}
|
|
|
|
VT_TEST(sparse_advise_dontneed_is_safe) {
|
|
TempPath tp;
|
|
SparseFile f;
|
|
VT_REQUIRE(f.open(tp.path, {.total_size = 8192}).has_value());
|
|
VT_CHECK(f.write_at(0, bytes("data")).has_value());
|
|
VT_CHECK(f.sync().has_value());
|
|
f.advise_dontneed(0, 4096); // must not crash / must be a no-op-safe call
|
|
f.advise_dontneed(0, 0);
|
|
f.close().value();
|
|
}
|
|
|
|
VT_TEST(sparse_move_transfers_fd) {
|
|
TempPath tp;
|
|
SparseFile a;
|
|
VT_REQUIRE(a.open(tp.path, {.total_size = 32}).has_value());
|
|
SparseFile b = std::move(a);
|
|
VT_CHECK(!a.is_open());
|
|
VT_CHECK(b.is_open());
|
|
VT_CHECK(b.write_at(0, bytes("moved")).has_value());
|
|
b.close().value();
|
|
}
|
|
|
|
VT_TEST(sparse_concurrent_nonoverlapping_writes) {
|
|
TempPath tp;
|
|
SparseFile f;
|
|
constexpr int kSegs = 8;
|
|
constexpr std::size_t kSeg = 64 * 1024;
|
|
VT_REQUIRE(f.open(tp.path, {.total_size = kSegs * kSeg}).has_value());
|
|
|
|
std::vector<std::jthread> ts;
|
|
for (int s = 0; s < kSegs; ++s) {
|
|
ts.emplace_back([&, s] {
|
|
std::vector<std::byte> chunk(kSeg, static_cast<std::byte>('A' + s));
|
|
for (std::size_t off = 0; off < kSeg; off += 4096) {
|
|
auto r = f.write_at(static_cast<std::uint64_t>(s) * kSeg + off,
|
|
vdm::ConstByteSpan(chunk.data() + off, 4096));
|
|
if (!r.has_value())
|
|
VT_FAIL("concurrent write_at failed");
|
|
}
|
|
});
|
|
}
|
|
ts.clear(); // join
|
|
VT_CHECK(f.sync().has_value());
|
|
|
|
auto data = read_all(tp.path);
|
|
VT_REQUIRE(data.size() == kSegs * kSeg);
|
|
for (int s = 0; s < kSegs; ++s) {
|
|
bool ok = true;
|
|
for (std::size_t i = 0; i < kSeg; ++i)
|
|
if (data[s * kSeg + i] != static_cast<std::byte>('A' + s))
|
|
ok = false;
|
|
VT_CHECK(ok);
|
|
}
|
|
f.close().value();
|
|
}
|