core: add the download engine — task machine, DownloadHandle, Engine
Stage 8 of the CORE build order: the bodies behind the DownloadSpec / callback API reviewed in core/docs/engine-api-m1.md. Wires probe -> segment workers -> WriteBuffer -> SparseFile -> .veloxpart.meta -> retry/backoff -> SegmentBudget -> RateLimiter -> callbacks into one event-driven machine. - Engine (src/engine.cpp): owns HttpClient, Prober, SegmentBudget, RateLimiter and one timer jthread (min-heap of scheduled fns). start() builds a task and returns a DownloadHandle; ~Impl quiesces every task before joining the timer so no callback fires during teardown. - DownloadTaskState (src/task/download_task.cpp): one `mu` task lock; a shared_mutex over the worker map for the curl write path; callbacks collected under `mu` and fired after release via a separate deferred queue; weak_from_this() in every async hop. State machine over the CORE-owned EngineState subset, auto-pause on 401/407 and on a 200 where 206 was expected, validated resume via If-Range. - digest (src/task/digest.cpp): OpenSSL EVP hash_file() for the optional post-download checksum; links OpenSSL::Crypto PRIVATE. - Segmenter::release_segment(): hand a paused segment back to the pool unassigned so resume's assign_slot() picks it up instead of splitting a still-"assigned" range and orphaning its front half. - DownloadHandle now names the real control block (vdm::task:: DownloadTaskState, defined only in the engine TU) via a namespace-scope fwd decl and a public-but-effectively-engine-only ctor, replacing the nested State/friend pair. Every public signature is unchanged; DAEMON (vdm-79) confirmed sched/ names only the public API. Fixes found while building the end-to-end suite (tests/task/engine_test.cpp, 9 cases against tools/testserver, green under ASan/UBSan and TSan): - a dropped connection lost its unflushed WriteBuffer tail while advance() had already counted those bytes as done -> a retry resumed past an unwritten hole. Flush on the failure path. - when the byte counters hit total while other workers were still live, teardown dropped their buffered tails. Now: cancel them and let each worker's own seg_finished drain it (the `assembling` state), last one starts verification -- no cross-thread buffer access. - seg_head() let a 401 with credentials present abort before libcurl's resend; now it proceeds once and acts on the final status. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01HPPSGhiArbvQgwC2DNiURS
This commit is contained in:
Claude Sonnet 5
@@ -34,6 +34,13 @@ vdm_add_test(veloxcore_engine_api_test task/api_compiles_test.cpp)
|
||||
vdm_add_test(veloxcore_token_bucket_test rate/token_bucket_test.cpp)
|
||||
|
||||
set(_testserver ${CMAKE_SOURCE_DIR}/tools/testserver/testserver.py)
|
||||
vdm_add_test(veloxcore_engine_test task/engine_test.cpp)
|
||||
target_include_directories(veloxcore_engine_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/net ${CMAKE_SOURCE_DIR}/core/src)
|
||||
if(EXISTS ${_testserver})
|
||||
target_compile_definitions(veloxcore_engine_test PRIVATE VDM_TESTSERVER_PY="${_testserver}")
|
||||
set_tests_properties(veloxcore_engine_test PROPERTIES TIMEOUT 300)
|
||||
endif()
|
||||
|
||||
foreach(net_it http_client probe)
|
||||
vdm_add_test(veloxcore_${net_it}_test net/${net_it}_test.cpp)
|
||||
target_include_directories(veloxcore_${net_it}_test
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
// End-to-end: a real Engine against tools/testserver, covering the CORE M1 DoD paths.
|
||||
|
||||
#include "vdm/engine.hpp"
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstdlib>
|
||||
#include <future>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "task/digest.hpp"
|
||||
#include "testserver_fixture.hpp"
|
||||
#include "vtest.hpp"
|
||||
|
||||
using namespace vdm;
|
||||
using namespace vdm::task;
|
||||
using vdm::testing::TestServer;
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
namespace {
|
||||
|
||||
struct TmpDir {
|
||||
std::string path;
|
||||
TmpDir() {
|
||||
const char *d = std::getenv("TMPDIR");
|
||||
path = (d ? d : "/tmp");
|
||||
path += "/vdm_engine_XXXXXX";
|
||||
path = ::mkdtemp(path.data()) ? path : "";
|
||||
}
|
||||
~TmpDir() {
|
||||
// best-effort recursive cleanup of our flat dir
|
||||
if (path.empty())
|
||||
return;
|
||||
std::string cmd = "rm -rf '" + path + "'";
|
||||
(void)std::system(cmd.c_str());
|
||||
}
|
||||
std::string file(const std::string &name) const { return path + "/" + name; }
|
||||
};
|
||||
|
||||
struct Recorder {
|
||||
std::promise<Result<DownloadOutcome>> done;
|
||||
std::future<Result<DownloadOutcome>> fut = done.get_future();
|
||||
std::atomic<bool> fired{false};
|
||||
std::vector<EngineState> states;
|
||||
std::mutex mu;
|
||||
std::atomic<int> auth_calls{0};
|
||||
std::atomic<int> decision_calls{0};
|
||||
// A probe callback can fire before the caller has stored the handle returned by
|
||||
// eng.start(). Callbacks that reach back into the handle wait on this.
|
||||
std::atomic<bool> handle_ready{false};
|
||||
|
||||
void arm(DownloadHandle &) { handle_ready.store(true, std::memory_order_release); }
|
||||
|
||||
DownloadCallbacks cbs(DownloadHandle *h = nullptr, std::string user = "",
|
||||
std::string pass = "") {
|
||||
DownloadCallbacks c;
|
||||
c.on_state = [this](EngineState, EngineState to, const std::optional<ErrorInfo> &) {
|
||||
std::lock_guard lk(mu);
|
||||
states.push_back(to);
|
||||
};
|
||||
c.on_finished = [this](Result<DownloadOutcome> r) {
|
||||
if (!fired.exchange(true))
|
||||
done.set_value(std::move(r));
|
||||
};
|
||||
if (h) {
|
||||
c.on_auth_required = [this, h, user, pass](const AuthChallenge &) {
|
||||
auth_calls.fetch_add(1);
|
||||
while (!handle_ready.load(std::memory_order_acquire))
|
||||
std::this_thread::sleep_for(1ms);
|
||||
h->provide_auth(user, pass, false);
|
||||
};
|
||||
}
|
||||
return c;
|
||||
}
|
||||
Result<DownloadOutcome> wait(std::chrono::seconds to = 40s) {
|
||||
if (fut.wait_for(to) != std::future_status::ready)
|
||||
return Err{Error::timeout, "engine test wait"};
|
||||
return fut.get();
|
||||
}
|
||||
bool saw(EngineState s) {
|
||||
std::lock_guard lk(mu);
|
||||
for (auto x : states)
|
||||
if (x == s)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
std::uint64_t file_size(const std::string &p) {
|
||||
int fd = ::open(p.c_str(), O_RDONLY);
|
||||
if (fd < 0)
|
||||
return ~0ull;
|
||||
off_t e = ::lseek(fd, 0, SEEK_END);
|
||||
::close(fd);
|
||||
return e < 0 ? ~0ull : static_cast<std::uint64_t>(e);
|
||||
}
|
||||
|
||||
// The reference SHA-256 the testserver will report for a given path.
|
||||
std::string server_sha(TestServer &srv, const std::string &mode, const std::string &size) {
|
||||
// one-shot GET /<mode>/sha256/<size> via a throwaway Engine::probe? no — use raw curl
|
||||
// through a small helper. Simplest: shell out.
|
||||
std::string url = srv.url("/" + mode + "/sha256/" + size);
|
||||
std::string cmd = "curl -s '" + url + "'";
|
||||
std::string out;
|
||||
if (FILE *f = ::popen(cmd.c_str(), "r")) {
|
||||
char buf[512];
|
||||
while (std::fgets(buf, sizeof buf, f))
|
||||
out += buf;
|
||||
::pclose(f);
|
||||
}
|
||||
auto q = out.find("\"sha256\"");
|
||||
if (q == std::string::npos)
|
||||
return {};
|
||||
auto colon = out.find(':', q);
|
||||
auto open = out.find('"', colon);
|
||||
auto close = out.find('"', open + 1);
|
||||
if (open == std::string::npos || close == std::string::npos)
|
||||
return {};
|
||||
return out.substr(open + 1, close - open - 1);
|
||||
}
|
||||
|
||||
DownloadSpec spec_for(TestServer &srv, const std::string &urlpath, const std::string &save) {
|
||||
DownloadSpec s;
|
||||
s.url = srv.url(urlpath);
|
||||
s.save_path = save;
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
VT_TEST(engine_plain_multisegment_download) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
VT_REQUIRE(!td.path.empty());
|
||||
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/plain/file/4M", td.file("a.bin")), rec.cbs());
|
||||
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(r.value().final_path, td.file("a.bin"));
|
||||
VT_CHECK_EQ(r.value().bytes, 4u * 1024 * 1024);
|
||||
VT_CHECK_EQ(file_size(td.file("a.bin")), 4u * 1024 * 1024);
|
||||
VT_CHECK(rec.saw(EngineState::downloading));
|
||||
VT_CHECK(rec.saw(EngineState::complete));
|
||||
|
||||
auto got = hash_file(td.file("a.bin"), Checksum::Algo::sha256);
|
||||
VT_REQUIRE(got.has_value());
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "plain", "4M"));
|
||||
// the sidecar is gone on success
|
||||
VT_CHECK_EQ(::access((td.file("a.bin") + ".veloxpart.meta").c_str(), F_OK), -1);
|
||||
}
|
||||
|
||||
VT_TEST(engine_checksum_pass_and_mismatch) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Engine eng;
|
||||
|
||||
std::string want = server_sha(srv, "plain", "1M");
|
||||
VT_REQUIRE(!want.empty());
|
||||
|
||||
{
|
||||
Recorder rec;
|
||||
auto s = spec_for(srv, "/plain/file/1M", td.file("ok.bin"));
|
||||
s.checksum = Checksum{Checksum::Algo::sha256, want};
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK(rec.saw(EngineState::verifying));
|
||||
}
|
||||
{
|
||||
Recorder rec;
|
||||
auto s = spec_for(srv, "/plain/file/1M", td.file("bad.bin"));
|
||||
s.checksum = Checksum{Checksum::Algo::sha256, std::string(64, 'a')};
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::checksum_mismatch);
|
||||
}
|
||||
}
|
||||
|
||||
VT_TEST(engine_non_resumable_single_segment) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/no-range/file/2M", td.file("nr.bin")), rec.cbs());
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(file_size(td.file("nr.bin")), 2u * 1024 * 1024);
|
||||
auto got = hash_file(td.file("nr.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "no-range", "2M"));
|
||||
}
|
||||
|
||||
VT_TEST(engine_404_is_an_error) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/plain/nope", td.file("x.bin")), rec.cbs());
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::not_found);
|
||||
VT_CHECK(rec.saw(EngineState::failed));
|
||||
}
|
||||
|
||||
VT_TEST(engine_cancel_mid_download) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/throttled/file/8M", td.file("c.bin")), rec.cbs());
|
||||
|
||||
for (int i = 0; i < 200 && !rec.saw(EngineState::downloading); ++i)
|
||||
std::this_thread::sleep_for(10ms);
|
||||
VT_REQUIRE(rec.saw(EngineState::downloading));
|
||||
h.cancel(/*discard_partial=*/true);
|
||||
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::canceled);
|
||||
VT_CHECK(rec.saw(EngineState::cancelled));
|
||||
VT_CHECK_EQ(::access((td.file("c.bin") + ".veloxpart").c_str(), F_OK), -1); // discarded
|
||||
}
|
||||
|
||||
VT_TEST(engine_pause_resume_completes) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/throttled/file/2M", td.file("pr.bin")), rec.cbs());
|
||||
|
||||
for (int i = 0; i < 200 && !rec.saw(EngineState::downloading); ++i)
|
||||
std::this_thread::sleep_for(10ms);
|
||||
h.pause();
|
||||
for (int i = 0; i < 100 && h.state() != EngineState::paused; ++i)
|
||||
std::this_thread::sleep_for(20ms);
|
||||
VT_CHECK_EQ(h.state(), EngineState::paused);
|
||||
h.resume();
|
||||
|
||||
auto r = rec.wait(90s);
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(file_size(td.file("pr.bin")), 2u * 1024 * 1024);
|
||||
auto got = hash_file(td.file("pr.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "throttled", "2M"));
|
||||
}
|
||||
|
||||
VT_TEST(engine_resume_after_a_fresh_task) {
|
||||
// Simulates kill -9: cancel WITHOUT discard, then a new task with allow_resume picks
|
||||
// up the .veloxpart[.meta] and finishes with a byte-identical file.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
std::string save = td.file("resume.bin");
|
||||
std::string want = server_sha(srv, "throttled", "3M");
|
||||
|
||||
{
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/throttled/file/3M", save), rec.cbs());
|
||||
for (int i = 0; i < 800 && (h.progress().downloaded < 512u * 1024); ++i)
|
||||
std::this_thread::sleep_for(10ms);
|
||||
VT_REQUIRE(h.progress().downloaded >= 512u * 1024);
|
||||
h.cancel(/*discard_partial=*/false);
|
||||
(void)rec.wait();
|
||||
VT_CHECK_EQ(::access((save + ".veloxpart.meta").c_str(), F_OK), 0); // sidecar kept
|
||||
}
|
||||
{
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto s = spec_for(srv, "/throttled/file/3M", save); // same source -> sidecar validates
|
||||
s.allow_resume = true;
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
auto r = rec.wait(120s);
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(file_size(save), 3u * 1024 * 1024);
|
||||
auto got = hash_file(save, Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), want); // byte-identical after resume
|
||||
}
|
||||
}
|
||||
|
||||
VT_TEST(engine_flaky_reset_retries_to_completion) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
// flaky-reset RSTs the first two attempts per (path,range); a single-segment request
|
||||
// therefore needs the retry loop.
|
||||
auto s = spec_for(srv, "/flaky-reset/file/256K", td.file("fl.bin"));
|
||||
s.segments = 1;
|
||||
s.max_retries = 40; // the server RSTs at the halfway point every attempt -> ~18 halvings
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
auto r = rec.wait(120s);
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(file_size(td.file("fl.bin")), 256u * 1024);
|
||||
auto got = hash_file(td.file("fl.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "flaky-reset", "256K"));
|
||||
VT_CHECK(rec.saw(EngineState::retry_wait) || rec.saw(EngineState::connecting));
|
||||
}
|
||||
|
||||
VT_TEST(engine_401_then_provide_auth_completes) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
DownloadHandle h;
|
||||
auto cbs = rec.cbs(&h, "test", "test");
|
||||
h = eng.start(spec_for(srv, "/401-basic/file/1M", td.file("au.bin")), std::move(cbs));
|
||||
rec.arm(h); // publish h to the auth callback (which may already be waiting)
|
||||
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK(rec.auth_calls.load() >= 1);
|
||||
VT_CHECK_EQ(file_size(td.file("au.bin")), 1u * 1024 * 1024);
|
||||
}
|
||||
Reference in New Issue
Block a user