capture.offer applies capture.enabled/excludedHosts/monitoredExtensions/
monitoredMimeTypes/minSizeBytes from settings, then the rules table (new
store::Rules + CORE's vdm::rules::match_rules/glob_match — DAEMON only converts its
own stored proto::Rule JSON into CORE's plain vocabulary, per that header's own
layering note), resolves the category folder (a rule's explicit categoryId/saveDir,
else store::Categories::guess_by_extension — the same extension guess
download.probe's suggestedCategoryId already used, now shared instead of
duplicated), dedupes against active (non-terminal) tasks by exact URL, and on `take`
calls add_one() — the same path download.add itself uses, so a captured download is
a real, admitted, persisted task, not a special case.
The 750ms deadline (CLAUDE.md §4 / AGENT-DAEMON.md build step 6) is checked
cooperatively between every step via a new rpc::CaptureDataSource seam: the real
implementation wraps store::Settings/Categories/Rules/Tasks; a test fake can jump its
own injected clock forward to simulate "the store was slow just now" with zero real
sleep. This catches the realistic failure mode (several slow steps adding up) though
it cannot preempt a single pathologically stuck call mid-flight — a true preemptive
guarantee would need the same async/background-thread treatment as download.probe,
which isn't safe to do against the same sqlite3 connection (opened SQLITE_OPEN_NOMUTEX,
explicitly not for concurrent use) without a second connection; left as a known,
documented limit of this pass rather than adding that plumbing speculatively.
capture.getRules returns the same settings-backed fields capture.offer itself reads,
so the two can never drift. rulesVersion is a placeholder constant (1) — no persisted
revision counter exists yet, and the extension already re-fetches on
event.settings.changed regardless.
New store/rules.{hpp,cpp}: rules table CRUD (list, and an atomic upsert+remove for
rules.upsert later). store::Categories::guess_by_extension replaces a duplicate copy
that used to live in sched/scheduler.cpp. store::Tasks::has_active_duplicate for the
dedupe check.
Verified against real veloxd + tools/testserver: a monitored-type offer answers in
~5ms and actually creates + downloads the task, correctly categorized; an
unmonitored type, an excluded host, a rule-vetoed host, and a second offer for a
still-active URL all answer ignore with the right reason; a bad category save dir
surfaces its real -32011. New capture_offer_test covers all of the above plus the
deadline itself (two cases, one per "slow" checkpoint), asserting real wall-clock
time barely moves even though the fake clock jumped 2 simulated seconds. Full ctest:
54/54 (excluding the pre-existing, unrelated conformance failure noted in the
previous commit).
Co-Authored-By: Claude Sonnet 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01GRDjHGgpYmMoPE2UFbe7pP
216 lines
8.8 KiB
C++
216 lines
8.8 KiB
C++
// capture.offer: excluded hosts, type/size filtering, rule matching, dedupe, take/ignore,
|
|
// and — the point of this file — the 750 ms deadline actually gets enforced when the
|
|
// store is slow, without a real sleep anywhere (a fake CaptureDataSource advances its own
|
|
// clock instead).
|
|
|
|
#include <chrono>
|
|
#include <string>
|
|
|
|
#include "check.hpp"
|
|
#include "rpc/capture_data_source.hpp"
|
|
#include "rpc/dispatcher.hpp"
|
|
#include "rpc/event_hub.hpp"
|
|
#include "store/migrations.hpp"
|
|
#include "store/sqlite.hpp"
|
|
#include "store/tasks.hpp"
|
|
#include "velox_proto.hpp"
|
|
|
|
using namespace velox::daemon;
|
|
namespace proto = velox::proto;
|
|
|
|
namespace {
|
|
|
|
// A controllable CaptureDataSource: every accessor returns a canned value, and `now()`
|
|
// reads a clock the test (or a "slow" accessor) can jump forward instantly. No real time
|
|
// ever passes — a test that jumps 2 real seconds still runs in microseconds.
|
|
class FakeCaptureSource : public rpc::CaptureDataSource {
|
|
public:
|
|
std::chrono::steady_clock::time_point clock = std::chrono::steady_clock::now();
|
|
bool enabled = true;
|
|
std::vector<std::string> ext = {"mp4"};
|
|
std::vector<std::string> mime;
|
|
std::int64_t min_size = 0;
|
|
std::vector<std::string> excluded;
|
|
std::vector<vdm::rules::Rule> rules_;
|
|
std::string category = "video";
|
|
std::string save_dir = "~/Downloads/velox-capture-test";
|
|
bool duplicate = false;
|
|
|
|
// Set to jump the clock forward by this much the next time the named accessor is
|
|
// called — simulates "this particular store read took a long time."
|
|
std::chrono::milliseconds slow_on_rules{0};
|
|
std::chrono::milliseconds slow_on_duplicate{0};
|
|
|
|
std::chrono::steady_clock::time_point now() override { return clock; }
|
|
bool capture_enabled() override { return enabled; }
|
|
std::vector<std::string> monitored_extensions() override { return ext; }
|
|
std::vector<std::string> monitored_mime_types() override { return mime; }
|
|
std::int64_t min_size_bytes() override { return min_size; }
|
|
std::vector<std::string> excluded_hosts() override { return excluded; }
|
|
std::vector<vdm::rules::Rule> enabled_rules() override {
|
|
clock += slow_on_rules;
|
|
return rules_;
|
|
}
|
|
std::string guess_category_id(const std::string&) override { return category; }
|
|
std::string category_save_dir(const std::string&) override { return save_dir; }
|
|
std::string default_save_dir() override { return save_dir; }
|
|
bool has_active_duplicate(const std::string&) override {
|
|
clock += slow_on_duplicate;
|
|
return duplicate;
|
|
}
|
|
};
|
|
|
|
proto::CaptureOfferParams offer(std::string url) {
|
|
proto::CaptureOfferParams p;
|
|
p.url = std::move(url);
|
|
p.method = proto::CaptureOfferParamsMethod::GET;
|
|
p.tabUrl = "https://example.com/page";
|
|
p.filename = std::string("movie.mp4");
|
|
p.contentType = std::string("video/mp4");
|
|
p.contentLength = 5'000'000;
|
|
return p;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
void run() {
|
|
auto db = store::Db::open(":memory:");
|
|
CHECK(db.has_value());
|
|
if (!db) return;
|
|
CHECK(store::migrate_to_head(*db).has_value());
|
|
|
|
rpc::EventHub hub;
|
|
rpc::VeloxDispatcher dispatcher(*db, hub);
|
|
|
|
// --- capture disabled --------------------------------------------------------------
|
|
{
|
|
FakeCaptureSource src;
|
|
src.enabled = false;
|
|
dispatcher.set_capture_source_for_test(&src);
|
|
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
|
CHECK(r.has_value());
|
|
if (r) {
|
|
CHECK(r->action == proto::CaptureOfferResultAction::Ignore);
|
|
CHECK(r->reason == proto::CaptureOfferResultReason::CaptureDisabled);
|
|
}
|
|
}
|
|
|
|
// --- excluded host -------------------------------------------------------------------
|
|
{
|
|
FakeCaptureSource src;
|
|
src.excluded = {"*.excluded.example"};
|
|
dispatcher.set_capture_source_for_test(&src);
|
|
auto r = dispatcher.on_capture_offer(offer("https://cdn.excluded.example/movie.mp4"));
|
|
CHECK(r.has_value());
|
|
if (r) {
|
|
CHECK(r->action == proto::CaptureOfferResultAction::Ignore);
|
|
CHECK(r->reason == proto::CaptureOfferResultReason::ExcludedHost);
|
|
}
|
|
}
|
|
|
|
// --- type not monitored --------------------------------------------------------------
|
|
{
|
|
FakeCaptureSource src;
|
|
src.ext = {"iso"}; // movie.mp4 doesn't match, and mime list is empty
|
|
dispatcher.set_capture_source_for_test(&src);
|
|
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
|
CHECK(r.has_value());
|
|
if (r) CHECK(r->reason == proto::CaptureOfferResultReason::TypeNotMonitored);
|
|
}
|
|
|
|
// --- below minimum size ----------------------------------------------------------
|
|
{
|
|
FakeCaptureSource src;
|
|
src.min_size = 10'000'000; // offer's contentLength is 5,000,000
|
|
dispatcher.set_capture_source_for_test(&src);
|
|
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
|
CHECK(r.has_value());
|
|
if (r) CHECK(r->reason == proto::CaptureOfferResultReason::BelowMinSize);
|
|
}
|
|
|
|
// --- a rule says ignore this host --------------------------------------------------
|
|
{
|
|
FakeCaptureSource src;
|
|
vdm::rules::Rule rule;
|
|
rule.rule_id = "r1";
|
|
rule.enabled = true;
|
|
rule.priority = 0;
|
|
rule.match.host_pattern = "cdn.example";
|
|
rule.action.capture = vdm::rules::CaptureVerdict::ignore;
|
|
src.rules_ = {rule};
|
|
dispatcher.set_capture_source_for_test(&src);
|
|
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
|
CHECK(r.has_value());
|
|
if (r) CHECK(r->reason == proto::CaptureOfferResultReason::RuleIgnore);
|
|
}
|
|
|
|
// --- duplicate -----------------------------------------------------------------------
|
|
{
|
|
FakeCaptureSource src;
|
|
src.duplicate = true;
|
|
dispatcher.set_capture_source_for_test(&src);
|
|
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
|
CHECK(r.has_value());
|
|
if (r) CHECK(r->reason == proto::CaptureOfferResultReason::Duplicate);
|
|
}
|
|
|
|
// --- take: a real task gets created, saved under the resolved category dir --------
|
|
{
|
|
FakeCaptureSource src;
|
|
dispatcher.set_capture_source_for_test(&src);
|
|
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
|
CHECK(r.has_value());
|
|
if (r) {
|
|
CHECK(r->action == proto::CaptureOfferResultAction::Take);
|
|
CHECK(r->taskId.has_value());
|
|
if (r->taskId) {
|
|
store::Tasks tasks(*db);
|
|
auto row = tasks.get(*r->taskId);
|
|
CHECK(row.has_value() && row->has_value());
|
|
if (row && *row) {
|
|
// resolve_target expands "~" and returns the canonical absolute path,
|
|
// not the literal string handed in.
|
|
CHECK((*row)->save_dir.find("velox-capture-test") != std::string::npos);
|
|
CHECK_EQ((*row)->category_id.value_or(""), std::string("video"));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- the deadline: a "slow" store still gets an answer, and it's `ignore` --------
|
|
// slow_on_rules jumps the fake clock forward 2 real seconds' worth the moment
|
|
// enabled_rules() is read (simulating a slow rules-table read); the 700 ms budget is
|
|
// long blown by the time the next checkpoint runs, so the offer must be ignored
|
|
// rather than proceeding to actually take the download. No real time passes — this
|
|
// whole test runs in microseconds.
|
|
{
|
|
FakeCaptureSource src;
|
|
src.slow_on_rules = std::chrono::milliseconds(2000);
|
|
dispatcher.set_capture_source_for_test(&src);
|
|
const auto wall_start = std::chrono::steady_clock::now();
|
|
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
|
const auto wall_elapsed = std::chrono::steady_clock::now() - wall_start;
|
|
CHECK(r.has_value());
|
|
if (r) {
|
|
CHECK(r->action == proto::CaptureOfferResultAction::Ignore);
|
|
CHECK(!r->taskId.has_value());
|
|
}
|
|
// The real wall clock barely moved — only the fake one jumped — proving the
|
|
// deadline check reads the injected clock, not a real sleep standing in for one.
|
|
CHECK(wall_elapsed < std::chrono::milliseconds(500));
|
|
}
|
|
|
|
// Same again, but the slow step is the dedupe check instead of rule matching — the
|
|
// deadline check has to run between every step, not just after one particular call.
|
|
{
|
|
FakeCaptureSource src;
|
|
src.slow_on_duplicate = std::chrono::milliseconds(2000);
|
|
dispatcher.set_capture_source_for_test(&src);
|
|
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
|
CHECK(r.has_value());
|
|
if (r) CHECK(r->action == proto::CaptureOfferResultAction::Ignore);
|
|
}
|
|
}
|
|
|
|
TEST_MAIN()
|