Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3e553f08c0 | ||
|
|
1d359af5a3 | ||
|
|
75536304bb | ||
|
|
27865888be | ||
|
|
939a19b7ea | ||
|
|
79c29b47e8 | ||
|
|
8e7d14ba7e | ||
|
|
c2eef96175 | ||
|
|
1fd2e0a0db | ||
|
|
755d85964e | ||
|
|
c6f864ea30 | ||
|
|
c1c5c82f8b | ||
|
|
4e177ec809 | ||
|
|
b5c6e1f47d | ||
|
|
0468b0176a | ||
|
|
d8b7c128be | ||
|
|
4f6c0cc9d2 | ||
|
|
e30d994d74 | ||
|
|
6632b75099 | ||
|
|
e0edf7a084 | ||
|
|
f4a6cebb3e | ||
|
|
963a76b6be | ||
|
|
765c1701d4 | ||
|
|
b578e6de1a | ||
|
|
ff721f4065 | ||
|
|
f2ee45f818 | ||
|
|
322a20efa5 | ||
|
|
7ea04fa79c | ||
|
|
ef58796d22 | ||
|
|
cf9e226e61 | ||
|
|
c89158ea09 | ||
|
|
4f6fb1029d | ||
|
|
55f0c6099d | ||
|
|
de83ee3cce | ||
|
|
39c69f3871 | ||
|
|
efe76c319d | ||
|
|
de748cc2fc | ||
|
|
67aefbce4b | ||
|
|
4df4d7ad08 | ||
|
|
f73a57a9ad | ||
|
|
c710481018 | ||
|
|
d4ad48d494 | ||
|
|
b60d4e6f5b | ||
|
|
6163898c14 | ||
|
|
ef816c21fb | ||
|
|
c99d1d9701 | ||
|
|
5b03eff926 | ||
|
|
03b6253b5a | ||
|
|
55932a2e11 | ||
|
|
25c171f742 | ||
|
|
2cb1959bff | ||
|
|
a967eca669 | ||
|
|
a71d904a1f | ||
|
|
41bce91770 | ||
|
|
0a38867579 | ||
|
|
0af4a5c4fc | ||
|
|
ba29fcb5bd | ||
|
|
824fa481bb |
@@ -18,8 +18,9 @@ policy so it can be re-applied or audited.
|
||||
| `bootstrap-script-2604` | now — real `--with-clang` install in a 26.04 container; the release the project ships on |
|
||||
| `build (gcc)` / `build (clang)` | now — core, daemon and gui have merged |
|
||||
| `sanitizers (dev)` / `sanitizers (tsan)` | now — core, daemon and gui have merged |
|
||||
| `conformance` | **now — `tests/conformance/` has landed; this is the M0 exit gate** |
|
||||
| `conformance` | **now — `tests/conformance/` has landed; this is the M0 exit gate.** Includes the live-`veloxd` runner (step 3b of `run.sh`), unconditional in the script — see `docs/adr/0019-live-veloxd-conformance-is-required.md`. |
|
||||
| `extension-lint` | now — `extension/` has merged (MV3 manifest + esbuild build) |
|
||||
| `gui-dod` | now — `gui/tests/dod/` has landed (GUI M1 DoD gates R3: `scroll-60fps`, `unhappy-path`); see `tests/integration/README.md#gui-m1-definition-of-done-gates-r3`. `gui-dod-nightly` (`rss-flat`) is schedule-only and cannot be a required PR check. |
|
||||
|
||||
`clang-tidy` is intentionally **not** required through M1 (`continue-on-error: true`,
|
||||
`.clang-tidy` has `WarningsAsErrors: ''`). Make it required at M2.
|
||||
|
||||
@@ -4,6 +4,9 @@ on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
schedule:
|
||||
- cron: '17 3 * * *' # nightly-integration only; every other job stays PR/push-triggered
|
||||
workflow_dispatch: # lets a human fire nightly-integration on demand
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -105,11 +108,16 @@ jobs:
|
||||
if: steps.check.outputs.present == 'true'
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: web-ext lint
|
||||
- name: eslint (no-download-logic gate + general rules)
|
||||
if: steps.check.outputs.present == 'true'
|
||||
working-directory: extension
|
||||
run: |
|
||||
npm ci
|
||||
npx eslint .
|
||||
- name: web-ext lint
|
||||
if: steps.check.outputs.present == 'true'
|
||||
working-directory: extension
|
||||
run: |
|
||||
npx web-ext lint --source-dir .
|
||||
|
||||
# --- build + test matrix ----------------------------------------------------------
|
||||
@@ -220,3 +228,76 @@ jobs:
|
||||
run: cmake --build --preset dev --target velox_conformance_cpp
|
||||
- name: Run conformance (ctest -L conformance)
|
||||
run: ctest --preset dev -L conformance --output-on-failure
|
||||
|
||||
nightly-integration:
|
||||
# Real veloxd + tools/testserver, 50 concurrent downloads mixing hostile modes,
|
||||
# every completed file's SHA-256 checked against testserver's own /sha256/ route,
|
||||
# veloxd's open-FD count checked flat across the run. Nightly, not per-PR: it's
|
||||
# ~2 minutes of real network I/O against a local server, not a schema check.
|
||||
# See tests/integration/README.md#nightly-integration-run for what each assertion
|
||||
# catches and the forced-failure transcript proving it isn't vacuous.
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Bootstrap toolchain
|
||||
run: sudo ./tools/bootstrap.sh
|
||||
- name: Configure
|
||||
run: cmake --preset dev
|
||||
- name: Build veloxd
|
||||
run: cmake --build --preset dev --target veloxd
|
||||
- name: Nightly integration run
|
||||
run: python3 tests/integration/nightly_run.py --veloxd build/dev/bin/veloxd --tasks 50 --timeout 180
|
||||
|
||||
gui-dod:
|
||||
# Per-PR GUI M1 DoD gates (gui/docs/pkg-qa-requests-m1.md R3): scroll-60fps and
|
||||
# unhappy-path. The 10-minute rss-flat gate is gui-dod-nightly, not here. GUI's
|
||||
# harness defaults QT_QPA_PLATFORM=offscreen itself, so no Xvfb/compositor needed.
|
||||
# See tests/integration/README.md#gui-m1-definition-of-done-gates-r3 for what each
|
||||
# gate catches and the forced-failure transcript proving it isn't vacuous.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Bootstrap toolchain
|
||||
run: sudo ./tools/bootstrap.sh
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22' # tools/mockd
|
||||
- name: Configure + build
|
||||
run: |
|
||||
cmake --preset dev
|
||||
cmake --build --preset dev --target gui-dod-harness
|
||||
- name: Install mockd
|
||||
run: cd tools/mockd && npm ci
|
||||
- name: Gates
|
||||
run: |
|
||||
gui/tests/dod/run.sh scroll-60fps --json scroll.json
|
||||
gui/tests/dod/run.sh unhappy-path --json unhappy.json
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: gui-dod-${{ github.run_id }}
|
||||
path: "*.json"
|
||||
|
||||
gui-dod-nightly:
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Bootstrap toolchain
|
||||
run: sudo ./tools/bootstrap.sh
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Configure + build
|
||||
run: |
|
||||
cmake --preset dev
|
||||
cmake --build --preset dev --target gui-dod-harness
|
||||
- run: cd tools/mockd && npm ci
|
||||
- name: RSS soak (10 min)
|
||||
run: gui/tests/dod/run.sh rss-flat --json rss.json
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: gui-dod-rss-${{ github.run_id }}
|
||||
path: rss.json
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include "client.hpp"
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_loop.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/uds_server.hpp"
|
||||
#include "store/migrations.hpp"
|
||||
#include "store/settings.hpp"
|
||||
@@ -59,8 +60,9 @@ void run() {
|
||||
CHECK(settings.set_raw("saveTo.defaultDir",
|
||||
"\"" + g_allowed_root + "\"").has_value());
|
||||
}
|
||||
rpc::VeloxDispatcher dispatcher(*db);
|
||||
rpc::UdsServer server(loop, dispatcher, server_sock);
|
||||
rpc::EventHub hub;
|
||||
rpc::VeloxDispatcher dispatcher(*db, hub);
|
||||
rpc::UdsServer server(loop, dispatcher, hub, server_sock);
|
||||
const auto ec = server.start();
|
||||
CHECK(!ec);
|
||||
if (ec) return;
|
||||
|
||||
@@ -506,7 +506,12 @@ def emit_field_parse(f: Field, indent: str) -> list[str]:
|
||||
f'{i} const auto it = j.find("{f.name}");']
|
||||
if f.optional:
|
||||
# Absent and null mean the same thing: the field is not set. A client that omits
|
||||
# a nullable field and one that sends null are treated identically on purpose.
|
||||
# a nullable field and one that sends null are treated identically on purpose --
|
||||
# correct for create-style params, where there is no existing value to distinguish
|
||||
# "never set" from "explicitly cleared". Patch-style fields need the distinction
|
||||
# (download.update's patch: "an explicit null clears a nullable field") and get an
|
||||
# opt-in exception via x-clearable per ADR 0018 (not implemented yet: this is the
|
||||
# decision record, not the generator change).
|
||||
o.append(f"{i} if (it != j.end() && !it->is_null()) {{")
|
||||
o += emit_value_parse(f.type, "(*it)", "val", "fp", i + " ")
|
||||
o.append(f"{i} out.{m} = std::move(val);")
|
||||
|
||||
@@ -21,7 +21,7 @@ fixtures/
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"name": "download.add — start an ISO now, into the Programs category",
|
||||
"name": "download.add — add an ISO for later, into the Programs category",
|
||||
"description": "Why this case is worth pinning.",
|
||||
"transport": "uds", // optional: replay only on this transport
|
||||
"requires": "...", // optional: a condition a plain server cannot produce
|
||||
@@ -81,3 +81,27 @@ cases in `tests/integration/`.
|
||||
correct response is *no response*: past 750 ms the extension must abandon the offer and let
|
||||
Firefox download normally. A download manager that eats downloads when its daemon is down
|
||||
is worse than no download manager.
|
||||
|
||||
## No fixture may pair a real external URL with `startMode: "now"`
|
||||
|
||||
This suite replays every fixture against a real, live `veloxd` (`tests/conformance/run.sh`),
|
||||
not just `mockd`. `mockd` never actually fetches anything, so it hid this for a while: a
|
||||
fixture with `startMode: "now"` (or `"queue"` into a running queue — anything that gets
|
||||
admitted to the scheduler right away) and a real, resolvable URL makes a **real** daemon
|
||||
actually start downloading it, for real, onto whatever machine runs the suite. This
|
||||
happened — twice, with `download.add.json` pointed at a ~6 GB Ubuntu ISO, straight into the
|
||||
developer's real `~/Downloads`.
|
||||
|
||||
The fix in each case is one of:
|
||||
- `startMode: "later"` — exercises the add path (validation, category assignment, the
|
||||
event) without ever handing the task to the engine;
|
||||
- a URL under `example.org`/`example.com` (IANA-reserved for exactly this, RFC 2606) —
|
||||
resolvable enough to validate as a URL, never a real download source;
|
||||
- `requires`, if the fixture's entire point needs a real transfer to fail in a specific way
|
||||
(see `errors/download.add.disk-full.json`) — skipped by default, so it only ever runs
|
||||
where the condition has actually been arranged.
|
||||
|
||||
A real `saveDir` gets the same treatment for the same reason: an absolute path like
|
||||
`/home/sami/Downloads/...` only means anything on the machine that fixture was written on.
|
||||
Omit `saveDir` and let `saveTo.defaultDir` apply, or use a relative-feeling path under a
|
||||
root the runner controls.
|
||||
|
||||
@@ -1,22 +1,23 @@
|
||||
{
|
||||
"name": "capture.offer — attachment on a monitored type is taken",
|
||||
"description": "Golden fixture. tests/conformance replays this against the real daemon AND the TS client. If either side drifts, this goes red before the lanes ever integrate.",
|
||||
"description": "Golden fixture. tests/conformance replays this against the real daemon AND the TS client. If either side drifts, this goes red before the lanes ever integrate. url is example.org (RFC 2606), not a real download source: 'take' against a real veloxd (tests/conformance/run.sh) admits a real task and hands it to the engine for real, and no fixture may do that against a real external URL. contentLength is a plausible-but-small 5 MiB rather than a real ISO's size: the 'Programs' category's saveDir is a migration-seeded builtin (~/Downloads/Programs, daemon/src/store/migrations/0001_initial.sql), not something an isolated test run's settings can redirect, so 'take' always sparse-preallocates into that real path on whatever machine runs this suite -- keeping the declared size small keeps that footprint trivial instead of a real ISO's worth of disk. transport is uds only: a real 'take' persists an active task, so replaying this same fixture again on a second live transport against the same daemon would correctly dedupe against it (capture.offer dedupes by exact URL) and get 'ignore' instead -- an artifact of replaying one fixture against one shared daemon over two transports, not a behaviour to golden.",
|
||||
"transport": "uds",
|
||||
"request": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 42,
|
||||
"method": "capture.offer",
|
||||
"params": {
|
||||
"url": "https://releases.ubuntu.com/26.04/ubuntu-26.04-desktop-amd64.iso",
|
||||
"url": "https://example.org/dl/ubuntu-26.04-desktop-amd64.iso",
|
||||
"method": "GET",
|
||||
"tabUrl": "https://releases.ubuntu.com/26.04/",
|
||||
"tabUrl": "https://example.org/26.04/",
|
||||
"headers": {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:154.0) Gecko/20100101 Firefox/154.0",
|
||||
"Referer": "https://releases.ubuntu.com/26.04/",
|
||||
"Referer": "https://example.org/26.04/",
|
||||
"Accept": "*/*"
|
||||
},
|
||||
"cookies": [],
|
||||
"contentType": "application/octet-stream",
|
||||
"contentLength": 6228541440,
|
||||
"contentLength": 5242880,
|
||||
"contentDisposition": "attachment; filename=\"ubuntu-26.04-desktop-amd64.iso\"",
|
||||
"filename": "ubuntu-26.04-desktop-amd64.iso",
|
||||
"origin": "moz-extension://11111111-2222-3333-4444-555555555555"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "download.add \u2014 start an ISO now, into the Programs category",
|
||||
"description": "The ordinary add path. saveDir is canonicalized and checked against the allowed roots before anything is written.",
|
||||
"name": "download.add — add an ISO for later, into the Programs category",
|
||||
"description": "The ordinary add path. saveDir is canonicalized and checked against the allowed roots before anything is written. startMode is 'later' deliberately: this suite replays against a real veloxd (tests/conformance/run.sh), and a real daemon given startMode 'now' would actually start fetching url for real. No fixture may pair a real external URL with startMode 'now' -- see contracts/fixtures/README.md.",
|
||||
"request": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 11,
|
||||
@@ -8,10 +8,9 @@
|
||||
"params": {
|
||||
"url": "https://releases.ubuntu.com/26.04/ubuntu-26.04-desktop-amd64.iso",
|
||||
"filename": "ubuntu-26.04-desktop-amd64.iso",
|
||||
"saveDir": "/home/sami/Downloads/Programs",
|
||||
"categoryId": "programs",
|
||||
"segments": 8,
|
||||
"startMode": "now"
|
||||
"startMode": "later"
|
||||
}
|
||||
},
|
||||
"response": {
|
||||
@@ -19,13 +18,13 @@
|
||||
"id": 11,
|
||||
"result": {
|
||||
"taskId": "$uuid",
|
||||
"state": "connecting",
|
||||
"state": "paused",
|
||||
"duplicate": null
|
||||
}
|
||||
},
|
||||
"assertions": [
|
||||
"the .veloxpart file is created sparse and preallocated at the final size",
|
||||
"saveDir resolves inside saveTo.allowedRoots, or the call fails -32011 having written nothing",
|
||||
"saveDir is omitted here on purpose: it resolves to saveTo.defaultDir, which is itself checked against saveTo.allowedRoots the same way an explicit saveDir would be -- see errors/download.add.invalid-path.json for the -32011 case",
|
||||
"startMode 'later' lands the task in 'paused' and never hands it to the engine, so nothing is fetched and no .veloxpart is created yet -- that only happens once the task is actually started (download.start.json, or startMode 'now'/'queue' against a source this suite controls)",
|
||||
"event.task.added is emitted to every subscriber before this reply is sent"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
],
|
||||
"defaults": {
|
||||
"url": "https://example.org/",
|
||||
"categoryId": "compressed",
|
||||
"categoryId": "programs",
|
||||
"startMode": "queue",
|
||||
"queueId": "main"
|
||||
}
|
||||
|
||||
@@ -30,5 +30,6 @@
|
||||
"the version check is transport-independent; this is replayed on the Unix socket so it is not masked by -32002",
|
||||
"data.expected is the daemon's own current protocol version string (kProtocolVersion), not a bare major and not pinnable in a golden file -- the conformance compare on error payloads is on `code` only, structural elsewhere, so echoing the live version is fine"
|
||||
],
|
||||
"transport": "uds"
|
||||
"transport": "uds",
|
||||
"closesConnection": true
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "limiter.get \u2014 the limiter is off",
|
||||
"description": "globalBps still carries the last configured value so the GUI can restore it when the user re-enables the limit.",
|
||||
"description": "globalBps still carries the last configured value so the GUI can restore it when the user re-enables the limit. applyToRunning is a write-only instruction on limiter.set (\"retune already-running transfers now\", not a persisted setting), so it never comes back from get.",
|
||||
"request": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 52,
|
||||
@@ -12,11 +12,11 @@
|
||||
"id": 52,
|
||||
"result": {
|
||||
"enabled": false,
|
||||
"globalBps": 2097152,
|
||||
"applyToRunning": false
|
||||
"globalBps": 2097152
|
||||
}
|
||||
},
|
||||
"assertions": [
|
||||
"enabled false means no throttling regardless of globalBps"
|
||||
"enabled false means no throttling regardless of globalBps",
|
||||
"applyToRunning is absent, not false: it's meaningless outside a limiter.set call"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -26,6 +26,9 @@ add_library(veloxcore STATIC
|
||||
src/task/digest.cpp
|
||||
src/task/download_task.cpp
|
||||
src/engine.cpp
|
||||
src/rules/filename.cpp
|
||||
src/rules/collision.cpp
|
||||
src/rules/match.cpp
|
||||
)
|
||||
add_library(velox::core ALIAS veloxcore)
|
||||
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
# M7 performance baseline
|
||||
|
||||
Measured against `docs/04-engine-design.md` §8's targets, via `tools/bench/vdm_bench`
|
||||
(see that file's header comment for the exact commands — reproduced below with their
|
||||
actual output). `--preset release`, this machine, 2026-09-11 through 2026-09-12. This is
|
||||
a baseline record, not a sign-off: one of the three numbers below is loopback-only rather
|
||||
than measured against a real 1 Gbit link, stated plainly rather than rounded away — see
|
||||
"Open gaps". The RSS number *did* fail the DoD line in the first pass through this file;
|
||||
it's since been root-caused and fixed (`docs/adr/0017`), not just re-measured.
|
||||
|
||||
## Commands and results
|
||||
|
||||
```
|
||||
$ cmake --preset release && cmake --build --preset release
|
||||
|
||||
$ bin/vdm_bench throughput --size 5G --require-mbps 940 --max-cpu-pct 8
|
||||
throughput: 5368709120 bytes in 2.99s
|
||||
throughput 14371.00 Mbps
|
||||
cpu 196.52 % of one core
|
||||
peak RSS 22.81 MiB
|
||||
```
|
||||
Against `tools/bench/support/local_server.hpp`'s busybox loopback server, not a real 1
|
||||
Gbit link — no such link was available to test against in this environment, so
|
||||
`--require-mbps`/`--max-cpu-pct` weren't meaningfully exercised here (loopback trivially
|
||||
clears 940 Mbps; the 196% CPU figure reflects driving a link far faster than 1 Gbit, not
|
||||
the 1-Gbit-saturated cost the target is about). This needs re-running against a real
|
||||
1 Gbit peer before it can stand as the actual M1/M7 sign-off number.
|
||||
|
||||
```
|
||||
$ bin/vdm_bench heap-profile --tasks 20 --task-size 4M --top 5
|
||||
segment budget: 32 live across tasks, engine reports total=32 active=32 starved=9
|
||||
heap-profile: peak RSS 45.41 MiB, 20 tasks, 8 segments assumed
|
||||
```
|
||||
Default `Config` (`default_segments=8`, `max_active_segments=32`, `default_buffer_bytes=1
|
||||
MiB`), default `--task-size 4M`. **Now clears the 60 MB line** (45.41 MiB), after the real
|
||||
fix below — root-caused, not just re-measured. Superseded the original `load` run's ~70
|
||||
MiB number quoted in earlier drafts of this file; see `docs/adr/0017`.
|
||||
|
||||
```
|
||||
$ bin/vdm_bench alloc-check --size 512M --window-s 2
|
||||
alloc-check: 4 allocations in 2.00s (budget 15)
|
||||
```
|
||||
Clears the no-allocation-on-the-hot-path bar (docs/agents/AGENT-CORE.md) comfortably.
|
||||
This number is *after* a real fix landed in the same change:
|
||||
`net::HttpClient::Impl::drain_commands` was constructing an (always-allocating, in
|
||||
libstdc++) `std::deque` on every worker-loop iteration regardless of whether any command
|
||||
was actually pending — once per curl_multi_poll wake, i.e. on the transfer hot path. Fixed
|
||||
by checking `w.queue.empty()` under the lock before touching `local` at all. Before the
|
||||
fix this bench reported thousands of allocations/sec under any sustained transfer.
|
||||
|
||||
## ASan / UBSan / TSan (M1 DoD: "20-task load test... clean")
|
||||
|
||||
- `--preset dev` (ASan+UBSan) and `--preset tsan`: the full `core/` test suite (40 ctest
|
||||
cases, including `veloxcore_engine_test`'s hostile-mode suite and `veloxcore_budget_test`)
|
||||
and all three `tools/bench` smoke tests pass clean on both presets.
|
||||
- Two real bugs were caught and fixed getting here:
|
||||
- `DownloadTaskState::quiesce()` (engine shutdown / `Engine`'s destructor) cleared the
|
||||
`workers` map synchronously right after issuing an async `transfer.cancel()`, racing
|
||||
the HttpClient worker thread's still-in-flight write callback into a heap-use-after-free
|
||||
on the segment's ring buffer — ASan-caught via `alloc-check`, which (by design) drops
|
||||
its `Engine` while a download is still active. Fixed by having `quiesce()` wait for
|
||||
each worker to drain itself through the same `seg_finished` path every other exit uses,
|
||||
instead of tearing the map down itself.
|
||||
- `SegWorker::speed_bps` (the polled-progress fix, see `engine_polled_progress_reports_
|
||||
nonzero_speed`) was written only by a segment's own curl callback and, before this
|
||||
session, only ever read from that same thread (`emit_progress_if_due`, called from the
|
||||
same callback) — safe without synchronization. Reading it from `snapshot_progress()`
|
||||
(any thread calling `DownloadHandle::progress()`) broke that invariant: `workers_mu`'s
|
||||
shared_lock protects the `workers` map's structure, not an individual `SegWorker`'s
|
||||
mutable fields. TSan-caught. Fixed with `std::atomic<double>` (relaxed: this is an
|
||||
informational EMA, nothing synchronizes real state on it) rather than adding a lock to
|
||||
the write side.
|
||||
- The `tools/bench load` ctest registration still runs at reduced concurrency
|
||||
(`--tasks 8 --segments 2`) under sanitizer presets (`tools/bench/CMakeLists.txt`) from
|
||||
when this was written against `docs/adr/0016`'s postscript — see `docs/adr/0017`'s "Open
|
||||
gaps" note: that straggler is now suspected to have been the *same* root cause as the RSS
|
||||
bug, not re-verified at the DoD's full shape under `--preset tsan` in this change.
|
||||
|
||||
## Open gaps
|
||||
|
||||
1. ~~RSS is ~70 MB against a 60 MB target~~ **Fixed — see `docs/adr/0017`.** Root cause
|
||||
was not, as first guessed here, an `ADR 0012` arithmetic gap or per-queued-handle curl
|
||||
overhead: `SegmentBudget::confirm_slot()` only checked a task's *own* target against
|
||||
its own held count, never the engine-wide `active_` sum, so it could (and under real
|
||||
20-task/8-segment contention, reliably did) admit segments well past
|
||||
`max_active_segments` — `heap-profile` caught it directly: `budget.active` reading
|
||||
56–86 against a `total` of 32. Fixed at the budget level (the one place that can
|
||||
actually enforce the invariant); `docs/adr/0012`'s own arithmetic was fine all along.
|
||||
2. **Throughput/CPU numbers are loopback-only.** No 1 Gbit link was available to test
|
||||
against; re-run `throughput --size 5G --require-mbps 940 --max-cpu-pct 8` against a real
|
||||
one before treating this as signed off.
|
||||
3. **`docs/adr/0016`**: `rate::RateLimiter`'s global-limit path (byte-rate pacing, a
|
||||
different subsystem from the segment-admission bug in `docs/adr/0017`) has no fairness
|
||||
ordering under heavy segment contention (a shared `TokenBucket`'s peek/commit race can
|
||||
starve a waiter indefinitely) — a real, separate, still-open gap for the "global
|
||||
bandwidth cap with many concurrent downloads" scenario.
|
||||
4. **The TSan-only load-test straggler** noted in `docs/adr/0016`'s postscript, found
|
||||
before `docs/adr/0017`'s fix landed: plausibly the *same* root cause (a segment denied
|
||||
admission with nothing to wake it, worse under TSan's slowdown widening the window a
|
||||
deferred yield can sit in) rather than the `CURLOPT_LOW_SPEED_TIME` guess that ADR
|
||||
originally offered — not reverified at the DoD's full 20-task/8-segment shape under
|
||||
`--preset tsan` in this change (the sanitizer-preset smoke registration still runs at
|
||||
reduced concurrency; see `tools/bench/CMakeLists.txt`). Worth re-running before treating
|
||||
it as closed.
|
||||
@@ -1,12 +1,12 @@
|
||||
# `libveloxcore` — public API
|
||||
|
||||
**Status: M1 in progress.** `util/`, `net/` (http_client, probe, url, content_disposition),
|
||||
`io/` (sparse_file, write_buffer), `meta/veloxpart`, and `segment/` (segmenter, budget)
|
||||
are landed. The **download entry point** — `vdm::Engine`, `vdm::task::DownloadSpec` /
|
||||
`DownloadHandle` / `DownloadCallbacks` — is sketched in `vdm/engine.hpp` and
|
||||
`vdm/task/download.hpp` and **out for DAEMON review**: see
|
||||
[`core/docs/engine-api-m1.md`](../../docs/engine-api-m1.md). Bodies land in CORE stage 8;
|
||||
build against the value types now.
|
||||
`io/` (sparse_file, write_buffer), `meta/veloxpart`, `segment/` (segmenter, budget),
|
||||
`rate/` (token_bucket), `task/`+`engine.hpp` (the download engine itself — `vdm::Engine`,
|
||||
`DownloadSpec`/`DownloadHandle`/`DownloadCallbacks`), and `rules/` (filename sanitization,
|
||||
collision policy, rule-table matching) are landed. `media/` is M4, not started — see
|
||||
[`core/docs/engine-api-m1.md`](../../docs/engine-api-m1.md) for the engine API's own
|
||||
DAEMON-review history.
|
||||
|
||||
Layering (CLAUDE.md §3): this library knows nothing about JSON, SQL, Qt, or RPC. Input is
|
||||
a spec value; output is bytes on disk plus typed callbacks. DAEMON projects engine state
|
||||
@@ -75,3 +75,44 @@ Sink interface — core does no I/O itself. `LogSink` abstract base; DAEMON inst
|
||||
via `set_log_sink()`, default discards. `CallbackSink` adapter (with a min-level filter).
|
||||
`VDM_LOG_{TRACE,DEBUG,INFO,WARN,ERROR}(category, fmt, args...)` — `std::format` syntax,
|
||||
only formatted when a sink is installed and wants the level.
|
||||
|
||||
---
|
||||
|
||||
## `rules/` — filename sanitization, collision policy, rule-table matching
|
||||
|
||||
Pure functions only: no I/O, no filesystem access, no notion of the wire `Rule` type or
|
||||
its JSON/SQL representation. DAEMON owns the rule table (storage, `rules.upsert`, the
|
||||
generated `Rule` type) and decodes it into the plain structs below before calling in.
|
||||
|
||||
### `vdm/rules/filename.hpp`
|
||||
|
||||
`sanitize_filename(raw, max_bytes = 255)` — turns a raw candidate (from
|
||||
`net::parse_content_disposition` or `net::url_filename`, neither of which is
|
||||
filesystem-safe by design — see their own headers) into one safe to create on ext4, APFS,
|
||||
and NTFS alike: strips separators/control bytes, folds NTFS-illegal characters to `_`,
|
||||
neutralizes reserved Windows device names (`CON`, `COM1`, ...), and clamps length on a
|
||||
UTF-8 boundary. Total: never empty, never throws. **Not** the path-traversal security
|
||||
boundary — that's DAEMON's `fs/safepath`, which runs after this and is the one that
|
||||
matters adversarially.
|
||||
|
||||
### `vdm/rules/collision.hpp`
|
||||
|
||||
`resolve_collision(desired, exists, policy, max_attempts = 1000)` — given an existence
|
||||
predicate (DAEMON supplies a real one; tests supply an in-memory set), finds the next free
|
||||
name Explorer/Finder-style (`"name (1).ext"`, `"name (2).ext"`, ...) under
|
||||
`CollisionPolicy::rename`, or returns `desired` unchanged under `::overwrite`. Never
|
||||
fabricates a guaranteed-unique name past `max_attempts` — returns the last candidate tried
|
||||
and leaves "still colliding" for the caller to treat as a real error.
|
||||
|
||||
### `vdm/rules/match.hpp`
|
||||
|
||||
`match_rules(rules, input) -> optional<RuleAction>` — the evaluation half of
|
||||
`contracts/schema/types/Rule.schema.json`: tries rules in ascending `priority` order
|
||||
(ties keep table order), skips disabled rows, returns the first whose every *present*
|
||||
match clause (`extensions`, `mime_types`, `host_pattern`, `url_pattern`,
|
||||
`min_size_bytes`/`max_size_bytes`) is satisfied — an absent clause is not a constraint,
|
||||
and a size clause never matches speculatively when `MatchInput::size_bytes` is still
|
||||
unknown (pre-probe). `std::nullopt` means no rule matched; the caller's own default
|
||||
category applies. `glob_match(pattern, text)` — the `*`/`?` matcher `host_pattern` and
|
||||
`url_pattern` both use, case-insensitive, bounded work even on a pathological
|
||||
all-`*` pattern (iterative, not recursive).
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
// vdm/rules/collision.hpp — when a chosen filename is already taken in the destination
|
||||
// directory, decide what to try next.
|
||||
//
|
||||
// Pure: takes an existence predicate rather than touching a filesystem itself, so it never
|
||||
// races what it's deciding about and stays testable without one. DAEMON (which owns the
|
||||
// actual directory listing / stat calls, downstream of its own fs/safepath gate) supplies
|
||||
// that predicate; a test supplies an in-memory set.
|
||||
//
|
||||
// This header compiles standalone.
|
||||
|
||||
#ifndef VDM_RULES_COLLISION_HPP
|
||||
#define VDM_RULES_COLLISION_HPP
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace vdm::rules {
|
||||
|
||||
enum class CollisionPolicy {
|
||||
rename, // try "name (1).ext", "name (2).ext", ... until one is free
|
||||
overwrite, // return `desired` unchanged — caller intends to replace what's there
|
||||
};
|
||||
|
||||
// Under `CollisionPolicy::rename`: calls `exists(candidate)` first with `desired` itself,
|
||||
// then with "<stem> (1)<ext>", "<stem> (2)<ext>", ... (Explorer/Finder-style, splitting
|
||||
// `desired` on its last '.' the same way `sanitize_filename`'s truncation does), returning
|
||||
// the first candidate for which it returns false. `exists` is never called with anything
|
||||
// but a single leaf name, never a path.
|
||||
//
|
||||
// `max_attempts` bounds a pathological `exists` that always returns true (this function
|
||||
// always returns — it is not fallible): once reached, the last candidate tried is returned
|
||||
// as-is, still possibly colliding. That is deliberately not papered over with a
|
||||
// fabricated-unique name (a timestamp suffix, say) — silently handing back a name nobody
|
||||
// asked for is exactly the kind of thing that turns into a mystery file days later; a
|
||||
// caller that hits the bound should treat it as a real error, not swallow it here.
|
||||
//
|
||||
// Under `CollisionPolicy::overwrite`, `exists` and `max_attempts` are unused — `desired`
|
||||
// is returned unchanged.
|
||||
[[nodiscard]] std::string resolve_collision(std::string_view desired,
|
||||
const std::function<bool(std::string_view)> &exists,
|
||||
CollisionPolicy policy = CollisionPolicy::rename,
|
||||
int max_attempts = 1000);
|
||||
|
||||
} // namespace vdm::rules
|
||||
|
||||
#endif // VDM_RULES_COLLISION_HPP
|
||||
@@ -0,0 +1,49 @@
|
||||
// vdm/rules/filename.hpp — turn a raw, untrusted filename candidate into one safe to
|
||||
// create on a real filesystem, cross-platform.
|
||||
//
|
||||
// This is NOT the path-traversal security boundary — that's DAEMON's fs/safepath (the
|
||||
// process's one canonicalize-and-verify-against-allowed-roots gate; see its own header
|
||||
// comment). This runs earlier and is cooperative, not adversarial-proof on its own: turn
|
||||
// whatever `net::parse_content_disposition` or `net::url_filename` handed back (see their
|
||||
// headers — neither fully sanitizes for the filesystem, by design; this is where that
|
||||
// finishes) into a *reasonable* candidate so an ordinary download doesn't needlessly
|
||||
// collide with a reserved device name, get silently mangled by NTFS, or get truncated
|
||||
// mid-extension by safepath's own leaf check.
|
||||
//
|
||||
// Total on hostile input: never throws, never asserts, never returns empty (falls back to
|
||||
// a generic name). No JSON, no SQL, no Qt, no RPC, no filesystem access (CLAUDE.md §3) —
|
||||
// pure string transformation.
|
||||
//
|
||||
// This header compiles standalone.
|
||||
|
||||
#ifndef VDM_RULES_FILENAME_HPP
|
||||
#define VDM_RULES_FILENAME_HPP
|
||||
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace vdm::rules {
|
||||
|
||||
// Sanitizes `raw` into a single path component safe to create on ext4, APFS, and NTFS
|
||||
// alike:
|
||||
// - strips path separators ('/' always; '\' too — Windows treats it as one) and collapses
|
||||
// any run of '.' that would otherwise still read as a traversal attempt ("..", "...")
|
||||
// down to a single '.', so a stripped-separator name can't reconstitute one
|
||||
// - strips C0 control bytes (incl. NUL) and DEL (0x7F)
|
||||
// - replaces the other NTFS-illegal characters (`< > : " | ? *`) with '_', so a name
|
||||
// that's fine on Linux doesn't silently fail to sync/export to a Windows-formatted
|
||||
// drive or SMB share
|
||||
// - strips trailing '.' and ' ' (both are NTFS traps: silently dropped by the Win32 API,
|
||||
// so "name." and "name" would otherwise collide invisibly on export)
|
||||
// - a reserved Windows device name (CON, PRN, AUX, NUL, COM1–9, LPT1–9), matched
|
||||
// case-insensitively against the part before the first '.' (or the whole name if
|
||||
// there's no '.'), gets a trailing '_' so it stops shadowing a device
|
||||
// - clamps to `max_bytes` (default 255, the common ext4/APFS/NTFS component limit),
|
||||
// cutting on a UTF-8 boundary and preferring to keep a short trailing extension intact
|
||||
// - empty, or entirely stripped down to nothing, falls back to "download"
|
||||
[[nodiscard]] std::string sanitize_filename(std::string_view raw, std::size_t max_bytes = 255);
|
||||
|
||||
} // namespace vdm::rules
|
||||
|
||||
#endif // VDM_RULES_FILENAME_HPP
|
||||
@@ -0,0 +1,91 @@
|
||||
// vdm/rules/match.hpp — the pure evaluation half of the rules engine
|
||||
// (contracts/schema/types/Rule.schema.json): given a rule table and what's known about one
|
||||
// candidate download, decide which rule fires and what it says to do.
|
||||
//
|
||||
// DAEMON owns the rule table itself — storage, `rules.upsert`/`rules.list`, the wire
|
||||
// `Rule` type generated from the contract. It decodes its own JSON/SQL representation into
|
||||
// the plain structs below and calls in; core never sees JSON, SQL, or the generated
|
||||
// protocol types (CLAUDE.md §3) — these structs mirror the contract's shape in CORE's own
|
||||
// vocabulary, the same relationship `util/error.hpp`'s `Error` has to the wire error codes.
|
||||
//
|
||||
// Pure and total: no I/O, no throw, no crash on any input (an empty table, an empty
|
||||
// pattern, a rule with every match clause absent).
|
||||
//
|
||||
// This header compiles standalone.
|
||||
|
||||
#ifndef VDM_RULES_MATCH_HPP
|
||||
#define VDM_RULES_MATCH_HPP
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
namespace vdm::rules {
|
||||
|
||||
enum class StartMode { now, later, queue }; // mirrors contracts' StartMode
|
||||
enum class CaptureVerdict { take, ignore }; // mirrors RuleAction.capture
|
||||
|
||||
// All present clauses must match; an absent clause is not a constraint
|
||||
// (contracts/schema/types/Rule.schema.json's own wording for RuleMatch).
|
||||
struct RuleMatch {
|
||||
std::optional<std::vector<std::string>> extensions; // no leading '.'; matched
|
||||
// case-insensitively
|
||||
std::optional<std::vector<std::string>> mime_types; // matched case-insensitively
|
||||
std::optional<std::string> host_pattern; // glob_match against the host
|
||||
std::optional<std::string> url_pattern; // glob_match against the whole URL
|
||||
std::optional<std::uint64_t> min_size_bytes; // inclusive
|
||||
std::optional<std::uint64_t> max_size_bytes; // inclusive
|
||||
};
|
||||
|
||||
// What to do with a matching download.
|
||||
struct RuleAction {
|
||||
std::optional<std::string> category_id;
|
||||
std::optional<std::string> save_dir;
|
||||
std::optional<std::string> queue_id;
|
||||
std::optional<std::uint32_t> segments;
|
||||
std::optional<StartMode> start_mode;
|
||||
std::optional<CaptureVerdict> capture;
|
||||
};
|
||||
|
||||
// One row of the rules engine.
|
||||
struct Rule {
|
||||
std::string rule_id;
|
||||
bool enabled = true;
|
||||
std::int64_t priority = 0; // lower runs first
|
||||
RuleMatch match;
|
||||
RuleAction action;
|
||||
};
|
||||
|
||||
// What's known about one candidate download, to match rules against. A field being empty
|
||||
// (not `std::nullopt` — these are plain strings, not optionals) means "unknown, matches no
|
||||
// non-empty clause that needs it" — e.g. `size_bytes` is absent pre-probe, so any rule with
|
||||
// a size clause simply doesn't match yet; the caller is expected to re-run matching once
|
||||
// the probe fills it in, same as DownloadSpec itself gets refined post-probe.
|
||||
struct MatchInput {
|
||||
std::string extension; // lowercased, no leading '.'; empty if none
|
||||
std::string mime_type; // lowercased; empty if unknown
|
||||
std::string host; // lowercased effective-URL host
|
||||
std::string url; // the whole effective URL
|
||||
std::optional<std::uint64_t> size_bytes;
|
||||
};
|
||||
|
||||
// Rules are tried in ascending `priority` order (ties broken by table order), skipping
|
||||
// disabled rows; the first whose every present match clause is satisfied wins.
|
||||
// `std::nullopt` means no rule matched — the caller falls back to its own default category
|
||||
// (this function has no notion of a default; that policy lives with the caller).
|
||||
[[nodiscard]] std::optional<RuleAction> match_rules(const std::vector<Rule> &rules,
|
||||
const MatchInput &input);
|
||||
|
||||
// Case-insensitive glob: '*' matches any run of characters including none, '?' matches
|
||||
// exactly one character. No character classes, no escaping — rule patterns are meant to
|
||||
// stay simple (contracts/schema/types/Rule.schema.json's own hostPattern/urlPattern
|
||||
// description gives only "*.example.com" as the example). Exposed on its own because
|
||||
// hostPattern and urlPattern are both just this against different text, and because it has
|
||||
// its own test table worth keeping separate from match_rules's.
|
||||
[[nodiscard]] bool glob_match(std::string_view pattern, std::string_view text) noexcept;
|
||||
|
||||
} // namespace vdm::rules
|
||||
|
||||
#endif // VDM_RULES_MATCH_HPP
|
||||
@@ -100,6 +100,10 @@ class SegmentBudget {
|
||||
std::uint32_t target = 0; // last published
|
||||
SlotTargetFn on_target;
|
||||
std::optional<SteadyTime> starved_since;
|
||||
// confirm_slot() was denied by the engine-wide cap (not by this task's own
|
||||
// target) and hasn't been retried since. Cleared on the task's next successful
|
||||
// confirm_slot(), however that retry was triggered. See release_slot()'s comment.
|
||||
bool waiting_for_slot = false;
|
||||
};
|
||||
|
||||
// A unit of deferred work: callbacks are copied out here so the public entry points
|
||||
@@ -109,7 +113,12 @@ class SegmentBudget {
|
||||
std::optional<std::pair<std::function<void(EngineBudget)>, EngineBudget>> notify_now;
|
||||
};
|
||||
|
||||
[[nodiscard]] std::vector<TaskId> priority_order_locked() const;
|
||||
Plan reallocate_locked();
|
||||
// Appends a retry hint for the highest-priority task with waiting_for_slot set (other
|
||||
// than `exclude`, the task whose release just freed this slot -- see release_slot()'s
|
||||
// comment) to `plan`, if one exists.
|
||||
void wake_one_waiter_locked(Plan &plan, TaskId exclude) const;
|
||||
static void run(Plan &p);
|
||||
[[nodiscard]] std::uint32_t effective_cap_locked(const Task &t) const;
|
||||
[[nodiscard]] EngineBudget snapshot_locked() const;
|
||||
|
||||
@@ -290,11 +290,20 @@ struct HttpClient::Impl {
|
||||
}
|
||||
|
||||
void drain_commands(Worker &w) {
|
||||
// Called every worker-loop iteration (run(), below) -- once per curl_multi_poll
|
||||
// wake, so once per socket-readiness event on the transfer hot path -- but commands
|
||||
// (add/pause/resume/cancel) are rare next to that. Check empty under the lock
|
||||
// *before* touching `local`: libstdc++'s std::deque allocates its map array on
|
||||
// default construction even with nothing pushed to it, so constructing one every
|
||||
// iteration just to usually swap nothing into it was an allocation on every poll
|
||||
// wake, not just on an actual command -- exactly what the curl-write-callback path
|
||||
// must never do (AGENT-CORE.md; caught by tools/bench's alloc-check).
|
||||
std::unique_lock lk(w.mu);
|
||||
if (w.queue.empty())
|
||||
return;
|
||||
std::deque<Command> local;
|
||||
{
|
||||
std::lock_guard lk(w.mu);
|
||||
local.swap(w.queue);
|
||||
}
|
||||
local.swap(w.queue);
|
||||
lk.unlock();
|
||||
for (auto &cmd : local) {
|
||||
auto &st = cmd.state;
|
||||
switch (cmd.kind) {
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
#include "vdm/rules/collision.hpp"
|
||||
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace vdm::rules {
|
||||
namespace {
|
||||
|
||||
// Split on the last '.', unless it's a leading dot (a dotfile like ".gitignore" has no
|
||||
// extension by this convention — matches sanitize_filename's own treatment). Returns
|
||||
// {stem, ext} where `ext` includes the leading '.' when present.
|
||||
std::pair<std::string_view, std::string_view> split_stem_ext(std::string_view name) {
|
||||
auto dot = name.rfind('.');
|
||||
if (dot == std::string_view::npos || dot == 0)
|
||||
return {name, {}};
|
||||
return {name.substr(0, dot), name.substr(dot)};
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string resolve_collision(std::string_view desired,
|
||||
const std::function<bool(std::string_view)> &exists,
|
||||
CollisionPolicy policy, int max_attempts) {
|
||||
if (policy == CollisionPolicy::overwrite)
|
||||
return std::string(desired);
|
||||
|
||||
if (!exists || !exists(desired))
|
||||
return std::string(desired);
|
||||
|
||||
auto [stem, ext] = split_stem_ext(desired);
|
||||
std::string candidate;
|
||||
for (int n = 1; n <= max_attempts; ++n) {
|
||||
candidate = std::string(stem) + " (" + std::to_string(n) + ")" + std::string(ext);
|
||||
if (!exists(candidate))
|
||||
return candidate;
|
||||
}
|
||||
return candidate; // still colliding; see the header comment on why this isn't hidden
|
||||
}
|
||||
|
||||
} // namespace vdm::rules
|
||||
@@ -0,0 +1,127 @@
|
||||
#include "vdm/rules/filename.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cctype>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace vdm::rules {
|
||||
namespace {
|
||||
|
||||
bool is_control_byte(unsigned char c) { return c < 0x20 || c == 0x7F; }
|
||||
|
||||
bool is_ntfs_illegal(char c) {
|
||||
switch (c) {
|
||||
case '<':
|
||||
case '>':
|
||||
case ':':
|
||||
case '"':
|
||||
case '|':
|
||||
case '?':
|
||||
case '*':
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// CON, PRN, AUX, NUL, COM1-9, LPT1-9 — Win32 device names, matched case-insensitively.
|
||||
// `stem` is already ASCII-only by the time this runs (everything else has been filtered),
|
||||
// so a byte-wise toupper is enough; no locale, no UTF-8 concerns.
|
||||
bool is_reserved_device_name(std::string_view stem) {
|
||||
static constexpr std::array<std::string_view, 4> kFixed = {"CON", "PRN", "AUX", "NUL"};
|
||||
std::string upper(stem);
|
||||
std::transform(upper.begin(), upper.end(), upper.begin(),
|
||||
[](unsigned char c) { return static_cast<char>(std::toupper(c)); });
|
||||
for (auto f : kFixed)
|
||||
if (upper == f)
|
||||
return true;
|
||||
if (upper.size() == 4 && (upper.starts_with("COM") || upper.starts_with("LPT")) &&
|
||||
upper[3] >= '1' && upper[3] <= '9')
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
// Backs `pos` up off any UTF-8 continuation bytes (10xxxxxx) so a byte-length cut never
|
||||
// splits a multi-byte codepoint. `pos` is a candidate cut index into `s`, 0 <= pos <=
|
||||
// s.size().
|
||||
std::size_t utf8_safe_cut(std::string_view s, std::size_t pos) {
|
||||
while (pos > 0 && (static_cast<unsigned char>(s[pos]) & 0xC0) == 0x80)
|
||||
--pos;
|
||||
return pos;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string sanitize_filename(std::string_view raw, std::size_t max_bytes) {
|
||||
// Pass 1: drop control bytes and path separators outright; fold the other
|
||||
// NTFS-illegal characters to '_'. Everything else (including non-ASCII UTF-8) passes
|
||||
// through untouched.
|
||||
std::string s;
|
||||
s.reserve(raw.size());
|
||||
for (char c : raw) {
|
||||
auto uc = static_cast<unsigned char>(c);
|
||||
if (is_control_byte(uc) || c == '/' || c == '\\')
|
||||
continue;
|
||||
s.push_back(is_ntfs_illegal(c) ? '_' : c);
|
||||
}
|
||||
|
||||
// Pass 2: collapse any run of 2+ '.' down to one, so stripped separators can't
|
||||
// reconstitute a ".." (or longer) traversal-looking sequence out of what's left.
|
||||
{
|
||||
std::string collapsed;
|
||||
collapsed.reserve(s.size());
|
||||
for (std::size_t i = 0; i < s.size(); ++i) {
|
||||
if (s[i] == '.' && i > 0 && collapsed.size() > 0 && collapsed.back() == '.')
|
||||
continue;
|
||||
collapsed.push_back(s[i]);
|
||||
}
|
||||
s = std::move(collapsed);
|
||||
}
|
||||
|
||||
// Pass 3: strip trailing '.' and ' ' — both are silently dropped by the Win32 API, so
|
||||
// leaving them lets two different requested names collide invisibly on export.
|
||||
while (!s.empty() && (s.back() == '.' || s.back() == ' '))
|
||||
s.pop_back();
|
||||
|
||||
if (s.empty())
|
||||
s = "download";
|
||||
|
||||
// Pass 4: reserved device name check, against the part before the first '.' (or the
|
||||
// whole name if there's none). The '_' goes right after the stem, before any
|
||||
// extension ("NUL.txt" -> "NUL_.txt"), so the result still looks like the same kind of
|
||||
// file rather than growing a spurious trailing character after its extension.
|
||||
{
|
||||
auto dot = s.find('.');
|
||||
std::string_view stem = dot != std::string::npos ? std::string_view(s).substr(0, dot)
|
||||
: std::string_view(s);
|
||||
if (is_reserved_device_name(stem))
|
||||
s.insert(stem.size(), "_");
|
||||
}
|
||||
|
||||
// Pass 5: clamp to max_bytes, UTF-8-safe, keeping a short trailing extension intact
|
||||
// where possible.
|
||||
if (s.size() > max_bytes) {
|
||||
std::string_view ext;
|
||||
if (auto dot = s.rfind('.'); dot != std::string::npos && dot > 0 && s.size() - dot <= 16)
|
||||
ext = std::string_view(s).substr(dot);
|
||||
if (ext.size() < max_bytes) {
|
||||
std::size_t stem_budget = max_bytes - ext.size();
|
||||
std::size_t cut = utf8_safe_cut(s, stem_budget);
|
||||
s = s.substr(0, cut) + std::string(ext);
|
||||
} else {
|
||||
s = s.substr(0, utf8_safe_cut(s, max_bytes));
|
||||
}
|
||||
// Re-strip: truncation can expose a new trailing '.'/' ' (e.g. the byte right
|
||||
// before the cut was itself a dot that pass 3 had no reason to touch).
|
||||
while (!s.empty() && (s.back() == '.' || s.back() == ' '))
|
||||
s.pop_back();
|
||||
if (s.empty())
|
||||
s = "download";
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace vdm::rules
|
||||
@@ -0,0 +1,98 @@
|
||||
#include "vdm/rules/match.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cstddef>
|
||||
#include <numeric>
|
||||
|
||||
namespace vdm::rules {
|
||||
namespace {
|
||||
|
||||
char lower_ascii(char c) { return static_cast<char>(std::tolower(static_cast<unsigned char>(c))); }
|
||||
|
||||
bool ieq(std::string_view a, std::string_view b) {
|
||||
return a.size() == b.size() &&
|
||||
std::equal(a.begin(), a.end(), b.begin(),
|
||||
[](char x, char y) { return lower_ascii(x) == lower_ascii(y); });
|
||||
}
|
||||
|
||||
// A rule's `extensions` entries are documented with no leading '.', but be lenient about
|
||||
// one showing up anyway (a hand-edited rule table, an older client) rather than let a
|
||||
// clause that never matches silently swallow a whole category.
|
||||
std::string_view strip_leading_dot(std::string_view s) {
|
||||
return (!s.empty() && s.front() == '.') ? s.substr(1) : s;
|
||||
}
|
||||
|
||||
bool match_clause(const RuleMatch &m, const MatchInput &in) {
|
||||
if (m.extensions) {
|
||||
bool any = std::any_of(m.extensions->begin(), m.extensions->end(), [&](const auto &e) {
|
||||
return ieq(strip_leading_dot(e), in.extension);
|
||||
});
|
||||
if (!any)
|
||||
return false;
|
||||
}
|
||||
if (m.mime_types) {
|
||||
bool any = std::any_of(m.mime_types->begin(), m.mime_types->end(),
|
||||
[&](const auto &t) { return ieq(t, in.mime_type); });
|
||||
if (!any)
|
||||
return false;
|
||||
}
|
||||
if (m.host_pattern && !glob_match(*m.host_pattern, in.host))
|
||||
return false;
|
||||
if (m.url_pattern && !glob_match(*m.url_pattern, in.url))
|
||||
return false;
|
||||
if (m.min_size_bytes) {
|
||||
if (!in.size_bytes || *in.size_bytes < *m.min_size_bytes)
|
||||
return false;
|
||||
}
|
||||
if (m.max_size_bytes) {
|
||||
if (!in.size_bytes || *in.size_bytes > *m.max_size_bytes)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool glob_match(std::string_view pattern, std::string_view text) noexcept {
|
||||
// Classic iterative wildcard match (single backtrack point at the most recent '*'), not
|
||||
// the naive recursive version — bounded work on any input, including a pattern that is
|
||||
// nothing but repeated '*'s against a long `text`.
|
||||
std::size_t p = 0, t = 0;
|
||||
std::size_t star = std::string_view::npos, mark = 0;
|
||||
while (t < text.size()) {
|
||||
if (p < pattern.size() && (pattern[p] == '?' || lower_ascii(pattern[p]) == lower_ascii(text[t]))) {
|
||||
++p;
|
||||
++t;
|
||||
} else if (p < pattern.size() && pattern[p] == '*') {
|
||||
star = p++;
|
||||
mark = t;
|
||||
} else if (star != std::string_view::npos) {
|
||||
p = star + 1;
|
||||
t = ++mark;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
while (p < pattern.size() && pattern[p] == '*') ++p;
|
||||
return p == pattern.size();
|
||||
}
|
||||
|
||||
std::optional<RuleAction> match_rules(const std::vector<Rule> &rules, const MatchInput &input) {
|
||||
std::vector<std::size_t> order(rules.size());
|
||||
std::iota(order.begin(), order.end(), 0);
|
||||
// Stable by construction: std::stable_sort keeps table order among equal priorities.
|
||||
std::stable_sort(order.begin(), order.end(), [&](std::size_t a, std::size_t b) {
|
||||
return rules[a].priority < rules[b].priority;
|
||||
});
|
||||
for (auto i : order) {
|
||||
const Rule &r = rules[i];
|
||||
if (!r.enabled)
|
||||
continue;
|
||||
if (match_clause(r.match, input))
|
||||
return r.action;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
} // namespace vdm::rules
|
||||
@@ -37,12 +37,10 @@ SegmentBudget::EngineBudget SegmentBudget::snapshot_locked() const {
|
||||
return EngineBudget{max_active_, active_, starved};
|
||||
}
|
||||
|
||||
// The two-pass fairness allocation. Recomputes every task's target from scratch (so a
|
||||
// live cap cut naturally produces target < held -> yield), diffs against the last
|
||||
// published target, and collects the callbacks to fire once mu_ is released.
|
||||
SegmentBudget::Plan SegmentBudget::reallocate_locked() {
|
||||
// Priority order: DAEMON's list first, then any registered task not in it (defensive;
|
||||
// "a running task absent from the list sorts last").
|
||||
// DAEMON's list first, then any registered task not in it (defensive; "a running task
|
||||
// absent from the list sorts last"). Shared by reallocate_locked() and
|
||||
// wake_one_waiter_locked(), which need the same priority ordering.
|
||||
std::vector<TaskId> SegmentBudget::priority_order_locked() const {
|
||||
std::vector<TaskId> order;
|
||||
order.reserve(tasks_.size());
|
||||
for (TaskId id : order_)
|
||||
@@ -51,6 +49,14 @@ SegmentBudget::Plan SegmentBudget::reallocate_locked() {
|
||||
for (const auto &[id, _] : tasks_)
|
||||
if (std::find(order.begin(), order.end(), id) == order.end())
|
||||
order.push_back(id);
|
||||
return order;
|
||||
}
|
||||
|
||||
// The two-pass fairness allocation. Recomputes every task's target from scratch (so a
|
||||
// live cap cut naturally produces target < held -> yield), diffs against the last
|
||||
// published target, and collects the callbacks to fire once mu_ is released.
|
||||
SegmentBudget::Plan SegmentBudget::reallocate_locked() {
|
||||
std::vector<TaskId> order = priority_order_locked();
|
||||
|
||||
std::unordered_map<TaskId, std::uint32_t> target;
|
||||
target.reserve(order.size());
|
||||
@@ -155,6 +161,7 @@ void SegmentBudget::deregister_task(TaskId id) {
|
||||
active_ -= it->second.held;
|
||||
tasks_.erase(it);
|
||||
plan = reallocate_locked();
|
||||
wake_one_waiter_locked(plan, id); // a departing task frees real slots too
|
||||
}
|
||||
run(plan);
|
||||
}
|
||||
@@ -182,6 +189,19 @@ bool SegmentBudget::confirm_slot(TaskId id) {
|
||||
Task &t = it->second;
|
||||
if (t.held >= t.target)
|
||||
return false; // target was cut in the race
|
||||
// reallocate_locked()'s pool math bounds sum(target) <= max_active_ *as computed*, but
|
||||
// that doesn't bound sum(held): a task can be legitimately over its own just-lowered
|
||||
// target for a while (yield deferred to a segment boundary, ADR 0011 A1), and another
|
||||
// task's target can correctly rise to claim that capacity before the first task has
|
||||
// physically released it. active_ is the one number that's always true regardless of
|
||||
// any task's target bookkeeping, so it's the backstop. Denials here are remembered
|
||||
// (waiting_for_slot) rather than left for the caller to somehow ask again at the right
|
||||
// moment -- see release_slot()'s wake_one_waiter_locked() call.
|
||||
if (active_ >= max_active_) {
|
||||
t.waiting_for_slot = true;
|
||||
return false;
|
||||
}
|
||||
t.waiting_for_slot = false;
|
||||
++t.held;
|
||||
++active_;
|
||||
if (snapshot_locked() != last_notified_) {
|
||||
@@ -191,6 +211,25 @@ bool SegmentBudget::confirm_slot(TaskId id) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Appends a retry hint for the highest-priority task with waiting_for_slot set (other
|
||||
// than `exclude`) to `plan`. Only ever wakes a task confirm_slot() actually turned away --
|
||||
// not just anyone below its target, which would also fire for tasks that are fairly,
|
||||
// correctly not entitled to more right now (see reallocate_locked()'s own target math).
|
||||
void SegmentBudget::wake_one_waiter_locked(Plan &plan, TaskId exclude) const {
|
||||
for (TaskId id : priority_order_locked()) {
|
||||
if (id == exclude)
|
||||
continue;
|
||||
auto it = tasks_.find(id);
|
||||
if (it == tasks_.end())
|
||||
continue;
|
||||
const Task &t = it->second;
|
||||
if (t.waiting_for_slot && t.on_target) {
|
||||
plan.targets.emplace_back(t.on_target, t.target);
|
||||
return; // exactly one freed slot, exactly one retry hint
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void SegmentBudget::release_slot(TaskId id) {
|
||||
Plan plan;
|
||||
{
|
||||
@@ -201,6 +240,11 @@ void SegmentBudget::release_slot(TaskId id) {
|
||||
--it->second.held;
|
||||
--active_;
|
||||
plan = reallocate_locked();
|
||||
// reallocate_locked() only fires a callback for a task whose *target* changed.
|
||||
// The task this freed slot is actually owed to (see confirm_slot()) may have a
|
||||
// target that was already correct and hasn't moved -- nothing else will ever ask
|
||||
// it to retry, so the budget has to remember and hand this slot to it directly.
|
||||
wake_one_waiter_locked(plan, id);
|
||||
}
|
||||
run(plan);
|
||||
}
|
||||
|
||||
+387
-88
@@ -15,6 +15,7 @@
|
||||
#include <atomic>
|
||||
#include <cctype>
|
||||
#include <cerrno>
|
||||
#include <condition_variable>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
@@ -63,8 +64,32 @@ std::string lower(std::string s) {
|
||||
return s;
|
||||
}
|
||||
|
||||
// scheme://host[:port] of `url`, with no path/query/fragment -- what docs/04 §7's "403
|
||||
// after redirect: retry once with the original referrer" retries with as the Referer
|
||||
// header. Empty on an unparseable URL (the caller just won't get a referrer retry).
|
||||
std::string origin_of(std::string_view url) {
|
||||
auto s = net::split_url(url);
|
||||
if (!s.valid)
|
||||
return {};
|
||||
std::string out = s.scheme + "://" + s.host;
|
||||
if (s.port)
|
||||
out += ":" + std::to_string(*s.port);
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// What to do once every worker has drained (see DownloadTaskState::begin_drain_locked).
|
||||
// A worker that hits one of these outcomes must not tear the task down itself: siblings may
|
||||
// still be mid-transfer, and cutting them off synchronously — cancel every worker, clear the
|
||||
// map, proceed — drops their buffered-but-unflushed bytes while segment_completed() already
|
||||
// counts those bytes as done. That's exactly the class of bug that makes a later resume skip
|
||||
// real data (seg_data() runs append() without workers_mu, so a sibling's buffer can't be
|
||||
// flushed safely from here anyway). Instead: cancel the siblings, remember what to do, and
|
||||
// let each one's own seg_finished (which already flushes on every exit path) run it once the
|
||||
// worker map is actually empty.
|
||||
enum class PendingAction { none, verify, fail, auto_pause, demote };
|
||||
|
||||
struct SegWorker {
|
||||
std::uint32_t seg_index = 0;
|
||||
net::Transfer transfer;
|
||||
@@ -76,14 +101,26 @@ struct SegWorker {
|
||||
bool needs_auth = false;
|
||||
bool wrong_status = false;
|
||||
bool range_bad = false;
|
||||
bool forbidden = false; // 403 -- docs/04 §7's "retry once with the original referrer"
|
||||
bool auth_handshake = false; // saw a 401/407 and let libcurl resend with credentials
|
||||
std::string resp_etag, resp_last_modified; // captured on a wrong_status 200, for demote
|
||||
std::optional<ErrorInfo> flush_error;
|
||||
|
||||
int retries = 0;
|
||||
|
||||
SteadyTime sample_at{};
|
||||
std::uint64_t sample_bytes = 0;
|
||||
double speed_bps = 0;
|
||||
// Written only by this segment's own curl callback (seg_data, sequential -- no lock
|
||||
// held across the update, by design: the transfer hot path takes no lock it doesn't
|
||||
// need). snapshot_progress() reads it from whatever thread calls
|
||||
// DownloadHandle::progress() (DAEMON's polling, or anyone else's), which workers_mu's
|
||||
// shared_lock does NOT cover -- that lock only protects the `workers` map's own
|
||||
// structure, not an individual SegWorker's mutable fields. atomic<double> (relaxed:
|
||||
// this is an approximate, informational EMA, not something anything synchronizes
|
||||
// real state on) keeps that read-from-any-thread safe without adding a lock to the
|
||||
// write side. Found by TSan the first time anything actually read this cross-thread
|
||||
// (engine_polled_progress_reports_nonzero_speed, added alongside the speed_bps fix).
|
||||
std::atomic<double> speed_bps{0};
|
||||
};
|
||||
|
||||
struct DownloadTaskState : std::enable_shared_from_this<DownloadTaskState> {
|
||||
@@ -95,6 +132,9 @@ struct DownloadTaskState : std::enable_shared_from_this<DownloadTaskState> {
|
||||
std::mutex mu;
|
||||
std::shared_mutex workers_mu;
|
||||
std::mutex deferred_mu;
|
||||
// Notified (holding mu) whenever seg_finished removes an entry from `workers`; quiesce()
|
||||
// waits on it instead of clearing the map itself — see quiesce()'s comment.
|
||||
std::condition_variable workers_drained_cv;
|
||||
|
||||
EngineState state = EngineState::probing;
|
||||
std::optional<ErrorInfo> last_error;
|
||||
@@ -109,6 +149,16 @@ struct DownloadTaskState : std::enable_shared_from_this<DownloadTaskState> {
|
||||
std::optional<std::uint64_t> total_size;
|
||||
std::string origin_host;
|
||||
|
||||
// docs/04 §7's "403 after redirect: retry once with the original referrer" -- many
|
||||
// CDNs 403 a bare/foreign Referer. Starts as spec.referrer (the browser's, verbatim);
|
||||
// start_worker_locked() sends this, not spec.referrer directly, so a 403 retry can
|
||||
// override it (to the download URL's own origin) without touching what the caller
|
||||
// actually asked for. referrer_retried bounds it to exactly once per task -- a second
|
||||
// 403 with a same-origin Referer already set is a real, honest failure
|
||||
// (Error::forbidden), not something a referrer swap can fix.
|
||||
std::string effective_referrer;
|
||||
bool referrer_retried = false;
|
||||
|
||||
std::unique_ptr<segment::Segmenter> seg;
|
||||
std::unique_ptr<io::SparseFile> file;
|
||||
std::unordered_map<std::uint32_t, std::unique_ptr<SegWorker>> workers;
|
||||
@@ -120,19 +170,25 @@ struct DownloadTaskState : std::enable_shared_from_this<DownloadTaskState> {
|
||||
|
||||
bool pause_requested = false;
|
||||
bool cancel_requested = false;
|
||||
bool assembling = false; // every byte received; draining live workers' buffers to disk
|
||||
bool discard_on_cancel = false;
|
||||
bool awaiting_auth = false;
|
||||
bool awaiting_decision = false;
|
||||
int max_retries = 10;
|
||||
|
||||
// Set by begin_drain_locked while waiting for sibling workers to drain; see
|
||||
// PendingAction above.
|
||||
PendingAction pending_action = PendingAction::none;
|
||||
std::optional<ErrorInfo> pending_error;
|
||||
bool pending_auth = false;
|
||||
bool pending_decision = false;
|
||||
|
||||
std::atomic<std::int64_t> last_progress_ns{0};
|
||||
SteadyTime started_at{};
|
||||
|
||||
std::vector<std::function<void()>> deferred;
|
||||
|
||||
DownloadTaskState(TaskHost &h, TaskId i, DownloadSpec s, DownloadCallbacks c)
|
||||
: host(h), id(i), spec(std::move(s)), cbs(std::move(c)) {}
|
||||
: host(h), id(i), spec(std::move(s)), cbs(std::move(c)), effective_referrer(spec.referrer) {}
|
||||
|
||||
// --- deferred callbacks -------------------------------------------------------------
|
||||
void defer(std::function<void()> fn) {
|
||||
@@ -188,6 +244,12 @@ struct DownloadTaskState : std::enable_shared_from_this<DownloadTaskState> {
|
||||
void on_probe_result(Result<net::ProbeResult> r);
|
||||
void finish_probe_locked();
|
||||
void apply_slot_target(std::uint32_t n);
|
||||
void fill_slots_locked();
|
||||
// Confirms a budget slot and starts a worker for `seg_idx` if nothing already covers
|
||||
// it. false on either failure (already running, or budget denied) -- the caller's own
|
||||
// fallback (fill_slots_locked's assign_slot() loop, retry_worker()'s early return)
|
||||
// stays the same either way. Caller holds mu.
|
||||
bool try_start_segment_locked(std::uint32_t seg_idx);
|
||||
void start_worker_locked(std::uint32_t seg_idx);
|
||||
void restart_probe(bool with_auth);
|
||||
|
||||
@@ -200,8 +262,8 @@ struct DownloadTaskState : std::enable_shared_from_this<DownloadTaskState> {
|
||||
void begin_verify_locked();
|
||||
void fail_locked(ErrorInfo e);
|
||||
void auto_pause_locked(ErrorInfo e, bool auth, bool decision);
|
||||
void cancel_all_transfers_locked();
|
||||
void start_assembly_locked();
|
||||
void demote_to_single_segment_locked();
|
||||
void begin_drain_locked(PendingAction action, ErrorInfo e, bool auth, bool decision);
|
||||
void finalize_cancel_locked();
|
||||
void write_sidecar_locked();
|
||||
void emit_progress_if_due();
|
||||
@@ -248,7 +310,7 @@ void DownloadTaskState::restart_probe(bool with_auth) {
|
||||
pr.url = spec.url;
|
||||
pr.headers = spec.headers;
|
||||
pr.cookies = spec.cookies;
|
||||
pr.referrer = spec.referrer;
|
||||
pr.referrer = effective_referrer;
|
||||
pr.user_agent = spec.user_agent;
|
||||
pr.proxy = spec.proxy;
|
||||
if (with_auth)
|
||||
@@ -261,12 +323,37 @@ void DownloadTaskState::restart_probe(bool with_auth) {
|
||||
}
|
||||
|
||||
void DownloadTaskState::on_probe_result(Result<net::ProbeResult> r) {
|
||||
bool retry_probe_with_referrer = false;
|
||||
{
|
||||
std::unique_lock lk(mu);
|
||||
if (retired.load() || is_terminal(state))
|
||||
return;
|
||||
if (!r.has_value()) {
|
||||
fail_locked(std::move(r).error());
|
||||
ErrorInfo e = std::move(r).error();
|
||||
// docs/04 §7's referrer retry applies here too: a probe (HEAD, or the
|
||||
// ranged-GET fallback when HEAD is refused -- probe.cpp) can be the request
|
||||
// that actually gets 403'd, before any segment worker exists to retry it
|
||||
// (net::Prober builds its own request from ProbeRequest::referrer, not
|
||||
// through start_worker_locked() -- see restart_probe()'s use of
|
||||
// effective_referrer below). Same one-shot bound via referrer_retried as the
|
||||
// worker-level retry (seg_finished's w->forbidden branch) shares.
|
||||
if (e.code == Error::forbidden && !referrer_retried) {
|
||||
referrer_retried = true;
|
||||
effective_referrer = origin_of(spec.url);
|
||||
retry_probe_with_referrer = true;
|
||||
} else if (e.code == Error::forbidden) {
|
||||
// Already retried with the origin referrer and still 403 -- not something
|
||||
// another blind retry fixes (an expired signed URL, a private resource).
|
||||
// Ask rather than fail outright, the same "ask, don't just fail" shape as
|
||||
// wrong_status/range_bad/the worker-level 403 branch: refresh_url() is a
|
||||
// no-op once the task is terminal, and tools/testserver's expiring-signed-
|
||||
// url mode (also a bare 403, indistinguishable from any other without
|
||||
// parsing the body -- CLAUDE.md §3, core never does) is meant to be
|
||||
// recovered exactly that way.
|
||||
auto_pause_locked(std::move(e), false, true);
|
||||
} else {
|
||||
fail_locked(std::move(e));
|
||||
}
|
||||
} else {
|
||||
probe = std::move(r).value();
|
||||
have_probe = true;
|
||||
@@ -281,6 +368,8 @@ void DownloadTaskState::on_probe_result(Result<net::ProbeResult> r) {
|
||||
}
|
||||
}
|
||||
flush_deferred();
|
||||
if (retry_probe_with_referrer)
|
||||
restart_probe(false);
|
||||
}
|
||||
|
||||
void DownloadTaskState::finish_probe_locked() {
|
||||
@@ -345,27 +434,67 @@ void DownloadTaskState::finish_probe_locked() {
|
||||
host.budget().set_want(id, want_slots());
|
||||
}
|
||||
|
||||
bool DownloadTaskState::try_start_segment_locked(std::uint32_t seg_idx) {
|
||||
if (workers.count(seg_idx))
|
||||
return false;
|
||||
if (!host.budget().confirm_slot(id))
|
||||
return false;
|
||||
start_worker_locked(seg_idx);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Start workers up to `slot_target`, given whatever the budget currently confirms. Shared
|
||||
// by apply_slot_target() (the budget's async callback, whenever the computed target
|
||||
// actually changes) and demote_to_single_segment_locked() -- the demoted target can
|
||||
// legitimately equal what it was before the rebuild (e.g. a download already down to its
|
||||
// last segment), in which case SegmentBudget::set_want() no-ops and the async callback
|
||||
// never fires, so nothing else would ever start the replacement worker.
|
||||
void DownloadTaskState::fill_slots_locked() {
|
||||
// A segment mid-backoff (SegState::stalled -- seg_finished's retry path: released its
|
||||
// slot, scheduled a retry_worker() timer, and gave up quietly if confirm_slot() denied
|
||||
// it then) has no live worker and never surfaces through assign_slot() -- it stays
|
||||
// assigned to whichever segment iteration created it, just not running. It's not
|
||||
// "fresh work" the loop below would ever find on its own. Every path that can free
|
||||
// budget capacity ends up here (apply_slot_target(), driven by SegmentBudget's
|
||||
// target-changed callback *and* its wait-list wakeup for a task whose target didn't
|
||||
// move -- see confirm_slot()/release_slot() in budget.cpp), so this is the one place
|
||||
// that needs to give a stalled segment another try, not every caller of
|
||||
// retry_worker(). Bounded by slot_target like the assign_slot() loop below; a segment
|
||||
// this doesn't get to keeps its own scheduled retry_worker() timer as a second chance.
|
||||
for (auto &v : seg->snapshot()) {
|
||||
if (workers.size() >= slot_target)
|
||||
break;
|
||||
if (v.state == segment::SegState::stalled)
|
||||
try_start_segment_locked(v.index);
|
||||
}
|
||||
while (workers.size() < slot_target) {
|
||||
auto s = seg->assign_slot();
|
||||
if (!s) {
|
||||
host.budget().set_want(id, static_cast<std::uint32_t>(workers.size()));
|
||||
break;
|
||||
}
|
||||
if (!host.budget().confirm_slot(id)) {
|
||||
seg->set_segment_state(*s, segment::SegState::idle);
|
||||
break;
|
||||
}
|
||||
start_worker_locked(*s);
|
||||
}
|
||||
// Mirrors retry_worker()'s own transition: a stalled-segment restart above can be the
|
||||
// thing that takes a retry_wait task back to actually transferring, same as connecting
|
||||
// does for a task starting up.
|
||||
if ((state == EngineState::connecting || state == EngineState::retry_wait) &&
|
||||
!workers.empty())
|
||||
transition(EngineState::downloading, std::nullopt);
|
||||
}
|
||||
|
||||
void DownloadTaskState::apply_slot_target(std::uint32_t n) {
|
||||
{
|
||||
std::unique_lock lk(mu);
|
||||
if (retired.load() || is_terminal(state) || pause_requested || cancel_requested ||
|
||||
awaiting_auth || awaiting_decision || assembling || !seg)
|
||||
awaiting_auth || awaiting_decision || pending_action != PendingAction::none || !seg)
|
||||
return;
|
||||
slot_target = n;
|
||||
while (workers.size() < slot_target) {
|
||||
auto s = seg->assign_slot();
|
||||
if (!s) {
|
||||
host.budget().set_want(id, static_cast<std::uint32_t>(workers.size()));
|
||||
break;
|
||||
}
|
||||
if (!host.budget().confirm_slot(id)) {
|
||||
seg->set_segment_state(*s, segment::SegState::idle);
|
||||
break;
|
||||
}
|
||||
start_worker_locked(*s);
|
||||
}
|
||||
if (state == EngineState::connecting && !workers.empty())
|
||||
transition(EngineState::downloading, std::nullopt);
|
||||
fill_slots_locked();
|
||||
}
|
||||
flush_deferred();
|
||||
}
|
||||
@@ -394,7 +523,7 @@ void DownloadTaskState::start_worker_locked(std::uint32_t seg_idx) {
|
||||
req.url = current_url();
|
||||
req.headers = spec.headers;
|
||||
req.cookies = spec.cookies;
|
||||
req.referrer = spec.referrer;
|
||||
req.referrer = effective_referrer;
|
||||
req.user_agent = spec.user_agent;
|
||||
req.proxy = spec.proxy;
|
||||
req.auth = spec.auth;
|
||||
@@ -464,12 +593,20 @@ net::DataAction DownloadTaskState::seg_head(std::uint32_t seg_idx, const net::Re
|
||||
// gets a 200 (the source has no Range support).
|
||||
if (resumable && total_size && *total_size > 0 && h.status == 200) {
|
||||
w->wrong_status = true;
|
||||
if (auto v = h.headers.get("ETag"))
|
||||
w->resp_etag.assign(*v);
|
||||
if (auto v = h.headers.get("Last-Modified"))
|
||||
w->resp_last_modified.assign(*v);
|
||||
return net::DataAction::abort;
|
||||
}
|
||||
if (h.status == 416) {
|
||||
w->range_bad = true;
|
||||
return net::DataAction::abort;
|
||||
}
|
||||
if (h.status == 403) {
|
||||
w->forbidden = true;
|
||||
return net::DataAction::abort;
|
||||
}
|
||||
if (h.status >= 400)
|
||||
return net::DataAction::abort;
|
||||
seg->set_segment_state(seg_idx, segment::SegState::downloading);
|
||||
@@ -510,7 +647,9 @@ net::DataAction DownloadTaskState::seg_data(std::uint32_t seg_idx, ConstByteSpan
|
||||
auto dt = std::chrono::duration<double>(now - w->sample_at).count();
|
||||
if (dt >= 0.5) {
|
||||
double inst = static_cast<double>(w->recv - w->sample_bytes) / dt;
|
||||
w->speed_bps = w->speed_bps == 0 ? inst : 0.7 * w->speed_bps + 0.3 * inst;
|
||||
double prev = w->speed_bps.load(std::memory_order_relaxed);
|
||||
w->speed_bps.store(prev == 0 ? inst : 0.7 * prev + 0.3 * inst,
|
||||
std::memory_order_relaxed);
|
||||
w->sample_at = now;
|
||||
w->sample_bytes = w->recv;
|
||||
}
|
||||
@@ -541,6 +680,7 @@ void DownloadTaskState::seg_finished(std::uint32_t seg_idx, Result<net::Transfer
|
||||
w = std::move(it->second);
|
||||
workers.erase(it);
|
||||
}
|
||||
workers_drained_cv.notify_all(); // quiesce() may be waiting for `workers` to empty out
|
||||
if (retired.load()) { // engine shutting down / already terminal — no more callbacks
|
||||
if (w->buf)
|
||||
(void)w->buf->flush();
|
||||
@@ -552,7 +692,7 @@ void DownloadTaskState::seg_finished(std::uint32_t seg_idx, Result<net::Transfer
|
||||
// (content-length-mismatch's honest-length lie, flaky-reset's tail, a proxy RST after
|
||||
// the last byte). If the segment is fully covered, that's a success.
|
||||
if (seg && !cancel_requested && !pause_requested && !w->needs_auth && !w->wrong_status &&
|
||||
!w->flush_error && !w->range_bad) {
|
||||
!w->flush_error && !w->range_bad && !w->forbidden) {
|
||||
const std::uint64_t len = seg->segment_end(seg_idx) - seg->segment_start(seg_idx) + 1;
|
||||
if (len != 0 && seg->segment_completed(seg_idx) >= len) {
|
||||
r = Result<net::TransferStats>(net::TransferStats{});
|
||||
@@ -590,48 +730,116 @@ void DownloadTaskState::seg_finished(std::uint32_t seg_idx, Result<net::Transfer
|
||||
}
|
||||
return done();
|
||||
}
|
||||
if (assembling) {
|
||||
// The file is fully received; this worker was cancelled so its buffered tail lands
|
||||
// on disk. advance() has already counted these bytes; the flush makes them durable.
|
||||
if (w->buf) {
|
||||
if (auto f = w->buf->flush(); !f.has_value()) {
|
||||
release_slot();
|
||||
fail_locked(std::move(f).error());
|
||||
return done();
|
||||
}
|
||||
}
|
||||
if (pending_action != PendingAction::none) {
|
||||
// A sibling already decided the task is finishing (verify / fail / auto-pause /
|
||||
// demote); this worker's own outcome no longer matters. Drain it like every other
|
||||
// exit path: flush its buffer so segment_completed() stays true to disk, then hand
|
||||
// off to whichever worker finds the map empty.
|
||||
if (w->buf)
|
||||
(void)w->buf->flush(); // best-effort: we're already tearing down for another
|
||||
// reason, and the pending action doesn't depend on
|
||||
// this segment reaching any particular state.
|
||||
seg->advance(seg_idx, w->base_completed + w->recv);
|
||||
release_slot();
|
||||
if (workers.empty())
|
||||
begin_verify_locked();
|
||||
if (workers.empty()) {
|
||||
PendingAction action = std::exchange(pending_action, PendingAction::none);
|
||||
ErrorInfo e = pending_error.value_or(ErrorInfo(Error::internal, ""));
|
||||
bool auth = pending_auth, decision = pending_decision;
|
||||
switch (action) {
|
||||
case PendingAction::verify:
|
||||
begin_verify_locked();
|
||||
break;
|
||||
case PendingAction::fail:
|
||||
fail_locked(e);
|
||||
break;
|
||||
case PendingAction::auto_pause:
|
||||
auto_pause_locked(e, auth, decision);
|
||||
break;
|
||||
case PendingAction::demote:
|
||||
demote_to_single_segment_locked();
|
||||
break;
|
||||
case PendingAction::none:
|
||||
break;
|
||||
}
|
||||
}
|
||||
return done();
|
||||
}
|
||||
if (w->needs_auth) {
|
||||
cancel_all_transfers_locked();
|
||||
release_slot();
|
||||
auto_pause_locked(ErrorInfo(Error::auth_required, "401/407", w->http_status), true, false);
|
||||
return done();
|
||||
}
|
||||
if (w->wrong_status) {
|
||||
cancel_all_transfers_locked();
|
||||
release_slot();
|
||||
auto_pause_locked(ErrorInfo(Error::server_file_changed, "200 where 206 expected"), false,
|
||||
true);
|
||||
// A 200 where 206 was expected is ambiguous: the file really changed (ask, don't
|
||||
// corrupt — docs/04 §5), or the server just stopped honouring Range for this
|
||||
// connection while it's still the same file (docs/04 §7: demote to 1 segment and
|
||||
// continue). ETag/Last-Modified from the 200 itself, compared against what the
|
||||
// probe recorded, is the only signal that tells them apart. Prefer ETag strictly
|
||||
// when both sides have one — same rule If-Range itself uses — and only fall back to
|
||||
// Last-Modified when there's no ETag to compare; a coarse (often second-resolution)
|
||||
// Last-Modified that happens to match is weak evidence next to a mismatching ETag.
|
||||
bool same_file;
|
||||
if (!probe.etag.empty() && !w->resp_etag.empty())
|
||||
same_file = probe.etag == w->resp_etag;
|
||||
else if (!probe.last_modified.empty() && !w->resp_last_modified.empty())
|
||||
same_file = probe.last_modified == w->resp_last_modified;
|
||||
else
|
||||
same_file = false; // no validator to compare -> can't prove it, ask
|
||||
if (same_file) {
|
||||
demote_to_single_segment_locked();
|
||||
} else {
|
||||
auto_pause_locked(ErrorInfo(Error::server_file_changed, "200 where 206 expected"),
|
||||
false, true);
|
||||
}
|
||||
return done();
|
||||
}
|
||||
if (w->flush_error) {
|
||||
ErrorInfo e = *w->flush_error;
|
||||
release_slot();
|
||||
if (e.code == Error::disk_full) {
|
||||
cancel_all_transfers_locked();
|
||||
auto_pause_locked(e, false, false);
|
||||
} else {
|
||||
fail_locked(e);
|
||||
}
|
||||
return done();
|
||||
}
|
||||
if (w->range_bad)
|
||||
r = Result<net::TransferStats>(ErrorInfo(Error::range_not_satisfiable, "416"));
|
||||
if (w->range_bad) {
|
||||
// 416 mid-download means our range metadata is stale (docs/04 §7): re-probe and
|
||||
// re-split rather than retrying the same now-invalid range until it exhausts.
|
||||
release_slot();
|
||||
auto_pause_locked(ErrorInfo(Error::range_not_satisfiable, "416"), false, true);
|
||||
return done();
|
||||
}
|
||||
if (w->forbidden) {
|
||||
// docs/04 §7: "403 after redirect: retry once with the original referrer -- many
|
||||
// CDNs require it." Bare/foreign Referer is the common cause; origin_of() rebuilds
|
||||
// it from the (possibly redirected) URL the response actually came from. Exactly
|
||||
// once per task, not a backoff series -- a second 403 with a same-origin Referer
|
||||
// already set isn't something another blind retry can fix (a private/expired
|
||||
// resource, an expiring signed URL past its window, ...). That's not necessarily
|
||||
// terminal, though: ask (same "ask, don't just fail outright" shape as
|
||||
// wrong_status/range_bad above) rather than fail_locked() outright, specifically
|
||||
// so DownloadHandle::refresh_url() -- do_refresh_url() is a no-op once the task is
|
||||
// terminal -- stays usable for the case tools/testserver's README pairs it with:
|
||||
// a caller that gets a fresh signed URL and hands it back.
|
||||
release_slot();
|
||||
if (!referrer_retried) {
|
||||
referrer_retried = true;
|
||||
effective_referrer = origin_of(current_url());
|
||||
seg->set_segment_state(seg_idx, segment::SegState::stalled);
|
||||
auto wp = weak_from_this();
|
||||
host.schedule(std::chrono::steady_clock::now(), [wp, seg_idx] {
|
||||
if (auto s = wp.lock())
|
||||
s->retry_worker(seg_idx);
|
||||
});
|
||||
if (workers.empty())
|
||||
transition(EngineState::retry_wait, std::nullopt);
|
||||
} else {
|
||||
auto_pause_locked(ErrorInfo(Error::forbidden, "403", w->http_status), false, true);
|
||||
}
|
||||
return done();
|
||||
}
|
||||
|
||||
if (!r.has_value()) {
|
||||
ErrorInfo e = std::move(r).error();
|
||||
@@ -695,17 +903,8 @@ void DownloadTaskState::seg_finished(std::uint32_t seg_idx, Result<net::Transfer
|
||||
else
|
||||
release_slot();
|
||||
|
||||
if (seg->all_complete()) {
|
||||
if (workers.empty()) {
|
||||
begin_verify_locked();
|
||||
} else {
|
||||
// Byte counters are satisfied, but other workers are still live and their
|
||||
// tails may only be in their buffers. Cancel them; each one's seg_finished
|
||||
// (this thread, once its curl worker has truly stopped) flushes via the
|
||||
// `assembling` branch, and the last starts verification.
|
||||
start_assembly_locked();
|
||||
}
|
||||
}
|
||||
if (seg->all_complete())
|
||||
begin_verify_locked(); // drain-aware: defers if other workers are still live
|
||||
return done();
|
||||
}
|
||||
|
||||
@@ -713,7 +912,7 @@ void DownloadTaskState::retry_worker(std::uint32_t seg_idx) {
|
||||
{
|
||||
std::unique_lock lk(mu);
|
||||
if (retired.load() || is_terminal(state) || pause_requested || cancel_requested ||
|
||||
assembling || !seg)
|
||||
pending_action != PendingAction::none || !seg)
|
||||
return;
|
||||
if (workers.count(seg_idx))
|
||||
return;
|
||||
@@ -729,6 +928,10 @@ void DownloadTaskState::retry_worker(std::uint32_t seg_idx) {
|
||||
}
|
||||
|
||||
void DownloadTaskState::begin_verify_locked() {
|
||||
if (!workers.empty()) {
|
||||
begin_drain_locked(PendingAction::verify, ErrorInfo(Error::internal, ""), false, false);
|
||||
return;
|
||||
}
|
||||
transition(EngineState::assembling, std::nullopt);
|
||||
transition(EngineState::verifying, std::nullopt);
|
||||
(void)file->sync();
|
||||
@@ -778,7 +981,10 @@ void DownloadTaskState::begin_verify_locked() {
|
||||
}
|
||||
|
||||
void DownloadTaskState::fail_locked(ErrorInfo e) {
|
||||
cancel_all_transfers_locked();
|
||||
if (!workers.empty()) {
|
||||
begin_drain_locked(PendingAction::fail, std::move(e), false, false);
|
||||
return;
|
||||
}
|
||||
if (file)
|
||||
(void)file->close();
|
||||
if (seg)
|
||||
@@ -800,6 +1006,10 @@ void DownloadTaskState::fail_locked(ErrorInfo e) {
|
||||
}
|
||||
|
||||
void DownloadTaskState::auto_pause_locked(ErrorInfo e, bool auth, bool decision) {
|
||||
if (!workers.empty()) {
|
||||
begin_drain_locked(PendingAction::auto_pause, std::move(e), auth, decision);
|
||||
return;
|
||||
}
|
||||
awaiting_auth = auth;
|
||||
awaiting_decision = decision;
|
||||
if (file)
|
||||
@@ -832,26 +1042,45 @@ void DownloadTaskState::auto_pause_locked(ErrorInfo e, bool auth, bool decision)
|
||||
}
|
||||
}
|
||||
|
||||
void DownloadTaskState::cancel_all_transfers_locked() {
|
||||
std::unique_lock wl(workers_mu);
|
||||
for (auto &[idx, w] : workers)
|
||||
w->transfer.cancel();
|
||||
workers.clear();
|
||||
}
|
||||
|
||||
// Every byte is received but some workers are still live; their buffered tails would be
|
||||
// lost if we dropped them here (seg_data() runs append() without workers_mu, so we cannot
|
||||
// safely flush another segment's buffer from under it). Just cancel them and let each
|
||||
// worker's own seg_finished drain it through the `assembling` branch once its curl worker
|
||||
// has stopped.
|
||||
void DownloadTaskState::start_assembly_locked() {
|
||||
assembling = true;
|
||||
transition(EngineState::assembling, std::nullopt);
|
||||
// Cancel every live worker and remember what to do once they've all drained through
|
||||
// seg_finished's PendingAction branch (see the enum's comment). Never clears `workers`
|
||||
// itself — each worker removes itself, flushed, when its own transfer actually completes.
|
||||
void DownloadTaskState::begin_drain_locked(PendingAction action, ErrorInfo e, bool auth,
|
||||
bool decision) {
|
||||
pending_action = action;
|
||||
pending_error = std::move(e);
|
||||
pending_auth = auth;
|
||||
pending_decision = decision;
|
||||
std::shared_lock wl(workers_mu);
|
||||
for (auto &[idx, w] : workers)
|
||||
w->transfer.cancel();
|
||||
}
|
||||
|
||||
// The 200-where-206-expected we just saw carried the same ETag/Last-Modified the probe
|
||||
// recorded: same file, the server (or this connection) just doesn't honour Range. Rebuild
|
||||
// as a single non-resumable segment covering the whole file and keep going with a plain
|
||||
// GET. It re-transfers bytes we may already have — there's no way to ask a Range-blind
|
||||
// server for a suffix — but it never truncates or discards what's on disk, and a source
|
||||
// that hasn't changed serves identical bytes, so the result is still byte-correct.
|
||||
void DownloadTaskState::demote_to_single_segment_locked() {
|
||||
if (!workers.empty()) {
|
||||
begin_drain_locked(PendingAction::demote, ErrorInfo(Error::internal, ""), false, false);
|
||||
return;
|
||||
}
|
||||
resumable = false;
|
||||
seg = std::make_unique<segment::Segmenter>(total_size.value_or(0), 1, /*resumable=*/false,
|
||||
host.config().min_segment_bytes);
|
||||
transition(EngineState::connecting, std::nullopt);
|
||||
if (registered) {
|
||||
// set_want() alone is not enough: if the demoted target happens to equal what it
|
||||
// was before the rebuild (e.g. this was already the last live segment), it's a
|
||||
// no-op and the async budget callback never fires. Drive slot assignment directly.
|
||||
slot_target = want_slots();
|
||||
host.budget().set_want(id, slot_target);
|
||||
fill_slots_locked();
|
||||
}
|
||||
}
|
||||
|
||||
void DownloadTaskState::finalize_cancel_locked() {
|
||||
if (file)
|
||||
(void)file->close();
|
||||
@@ -915,10 +1144,11 @@ void DownloadTaskState::emit_progress_if_due() {
|
||||
std::shared_lock lk(workers_mu);
|
||||
double agg = 0;
|
||||
for (auto &[idx, w] : workers) {
|
||||
agg += w->speed_bps;
|
||||
double speed = w->speed_bps.load(std::memory_order_relaxed);
|
||||
agg += speed;
|
||||
SegmentProgress sp;
|
||||
sp.index = idx;
|
||||
sp.speed_bps = static_cast<std::uint64_t>(w->speed_bps);
|
||||
sp.speed_bps = static_cast<std::uint64_t>(speed);
|
||||
p.segments.push_back(sp);
|
||||
}
|
||||
p.speed_bps = static_cast<std::uint64_t>(agg);
|
||||
@@ -1025,7 +1255,10 @@ void DownloadTaskState::do_decide(Decision d) {
|
||||
return;
|
||||
awaiting_decision = false;
|
||||
if (d == Decision::abort) {
|
||||
fail_locked(ErrorInfo(Error::server_file_changed, "user aborted"));
|
||||
// Surface the reason the decision was actually asked for (range_metadata_stale
|
||||
// sets last_error to range_not_satisfiable, server_file_changed to itself), not
|
||||
// a hardcoded label that would misreport a 416 as a changed file.
|
||||
fail_locked(last_error.value_or(ErrorInfo(Error::server_file_changed, "user aborted")));
|
||||
} else {
|
||||
if (d == Decision::restart) {
|
||||
::unlink(part_path.c_str());
|
||||
@@ -1066,6 +1299,7 @@ void DownloadTaskState::do_refresh_url(std::string url, std::vector<net::HeaderF
|
||||
net::ProbeRequest pr;
|
||||
pr.url = spec.url;
|
||||
pr.headers = spec.headers;
|
||||
pr.referrer = effective_referrer;
|
||||
pr.auth = spec.auth;
|
||||
pr.proxy = spec.proxy;
|
||||
host.probe(std::move(pr), [wp](Result<net::ProbeResult> r) {
|
||||
@@ -1075,10 +1309,43 @@ void DownloadTaskState::do_refresh_url(std::string url, std::vector<net::HeaderF
|
||||
std::unique_lock lk(s->mu);
|
||||
if (s->retired.load() || is_terminal(s->state))
|
||||
return;
|
||||
if (r.has_value()) {
|
||||
s->probe.effective_url = r.value().effective_url;
|
||||
s->probe.etag = r.value().etag;
|
||||
s->probe.last_modified = r.value().last_modified;
|
||||
if (!r.has_value()) {
|
||||
lk.unlock();
|
||||
s->flush_deferred();
|
||||
return; // still paused; the caller can retry refresh_url() or decide()
|
||||
}
|
||||
if (!s->have_probe) {
|
||||
// The task's *first* probe never succeeded (e.g. this session's own
|
||||
// expiring-signed-url path: 403, one referrer retry, still 403 -> ask rather
|
||||
// than fail outright -- see on_probe_result() -- specifically so this branch
|
||||
// exists to recover it). finish_probe_locked() is what actually registers the
|
||||
// task with the budget and builds its Segmenter; nothing downstream of a
|
||||
// partial field copy would ever start a worker without it.
|
||||
s->probe = std::move(r).value();
|
||||
s->have_probe = true;
|
||||
s->awaiting_auth = false;
|
||||
s->awaiting_decision = false;
|
||||
s->finish_probe_locked();
|
||||
lk.unlock();
|
||||
s->flush_deferred();
|
||||
return;
|
||||
}
|
||||
s->probe.effective_url = r.value().effective_url;
|
||||
s->probe.etag = r.value().etag;
|
||||
s->probe.last_modified = r.value().last_modified;
|
||||
// refresh_url()'s own contract is "on a live OR PAUSED task, without losing
|
||||
// progress" -- distinct from do_decide(restart), which discards progress. A task
|
||||
// can be paused here for any of three reasons (a plain user pause, awaiting_auth,
|
||||
// or awaiting_decision -- e.g. this session's own 403-after-referrer-retry path,
|
||||
// or the pre-existing wrong_status/range_bad ones); apply_slot_target()'s guard
|
||||
// blocks on awaiting_auth/awaiting_decision specifically, so leaving either set
|
||||
// would have set_want() below recompute a target that nothing ever acts on --
|
||||
// the caller's new URL re-probed successfully and then the task just sat there.
|
||||
// Clear both and leave `paused` the same way do_decide(restart) does.
|
||||
if (s->state == EngineState::paused) {
|
||||
s->awaiting_auth = false;
|
||||
s->awaiting_decision = false;
|
||||
s->transition(EngineState::connecting, std::nullopt);
|
||||
}
|
||||
if (s->registered)
|
||||
s->host.budget().set_want(s->id, s->want_slots());
|
||||
@@ -1088,12 +1355,24 @@ void DownloadTaskState::do_refresh_url(std::string url, std::vector<net::HeaderF
|
||||
}
|
||||
|
||||
void DownloadTaskState::quiesce() {
|
||||
std::lock_guard lk(mu);
|
||||
std::unique_lock lk(mu);
|
||||
retired.store(true);
|
||||
std::unique_lock wl(workers_mu);
|
||||
for (auto &[idx, w] : workers)
|
||||
w->transfer.cancel();
|
||||
workers.clear();
|
||||
{
|
||||
std::shared_lock wl(workers_mu);
|
||||
for (auto &[idx, w] : workers)
|
||||
w->transfer.cancel();
|
||||
}
|
||||
// Do not clear `workers` here: transfer.cancel() only requests the HttpClient worker
|
||||
// thread stop the transfer, asynchronously -- it does not wait for that to happen. A
|
||||
// worker's SegWorker (and its WriteBuffer) may still be in active use by a curl write
|
||||
// callback running on that other thread right now. Clearing the map out from under it
|
||||
// was a real, ASan-caught heap-use-after-free (ring buffer freed here while
|
||||
// SparseFile::write_at() on the HttpClient worker thread was still writing through it).
|
||||
// Every worker removes and flushes itself, safely, via seg_finished once HttpClient
|
||||
// actually confirms the transfer has stopped (same path every other exit uses; see the
|
||||
// `retired` branch there) -- just wait for that to happen for all of them. Bounded by
|
||||
// however long a cancelled curl transfer takes to unwind, not user-controllable.
|
||||
workers_drained_cv.wait(lk, [this] { return workers.empty(); });
|
||||
}
|
||||
|
||||
EngineState DownloadTaskState::snapshot_state() {
|
||||
@@ -1104,6 +1383,26 @@ EngineState DownloadTaskState::snapshot_state() {
|
||||
Progress DownloadTaskState::snapshot_progress() {
|
||||
Progress p;
|
||||
std::lock_guard lk(mu);
|
||||
// Per-segment instantaneous speed lives on the live SegWorker (seg_data's 0.5s-sampled
|
||||
// EMA, see the `speed_bps` update below) -- a segment with no live worker (idle,
|
||||
// paused, complete, failed) has no speed to report and stays at SegmentProgress's
|
||||
// default 0. Read every live worker's speed up front so the seg->snapshot() loop below
|
||||
// (which covers *every* segment, not just live ones -- unlike emit_progress_if_due's
|
||||
// push-callback version, which only ever reports the segments it currently has
|
||||
// workers for) can look each one up by index.
|
||||
double agg_speed = 0;
|
||||
std::unordered_map<std::uint32_t, double> worker_speed;
|
||||
{
|
||||
std::shared_lock wl(workers_mu);
|
||||
worker_speed.reserve(workers.size());
|
||||
for (auto &[idx, w] : workers) {
|
||||
double speed = w->speed_bps.load(std::memory_order_relaxed);
|
||||
worker_speed.emplace(idx, speed);
|
||||
agg_speed += speed;
|
||||
}
|
||||
p.effective_segments = static_cast<std::uint32_t>(workers.size());
|
||||
}
|
||||
p.speed_bps = static_cast<std::uint64_t>(agg_speed);
|
||||
if (seg) {
|
||||
p.downloaded = seg->downloaded();
|
||||
for (auto &v : seg->snapshot()) {
|
||||
@@ -1113,15 +1412,15 @@ Progress DownloadTaskState::snapshot_progress() {
|
||||
sp.end = v.end;
|
||||
sp.completed = v.completed;
|
||||
sp.state = v.state;
|
||||
if (auto it = worker_speed.find(v.index); it != worker_speed.end())
|
||||
sp.speed_bps = static_cast<std::uint64_t>(it->second);
|
||||
p.segments.push_back(sp);
|
||||
}
|
||||
}
|
||||
p.total = total_size;
|
||||
{
|
||||
std::shared_lock wl(workers_mu);
|
||||
p.effective_segments = static_cast<std::uint32_t>(workers.size());
|
||||
}
|
||||
p.effective_buffer_bytes = effective_buffer;
|
||||
if (p.speed_bps > 0 && total_size && *total_size > p.downloaded)
|
||||
p.eta_seconds = static_cast<std::uint32_t>((*total_size - p.downloaded) / p.speed_bps);
|
||||
return p;
|
||||
}
|
||||
|
||||
|
||||
@@ -55,3 +55,7 @@ if(NOT EXISTS ${_testserver})
|
||||
message(STATUS "veloxcore: tools/testserver not present; net integration tests will "
|
||||
"skip their server-backed cases.")
|
||||
endif()
|
||||
|
||||
vdm_add_test(veloxcore_rules_filename_test rules/filename_test.cpp)
|
||||
vdm_add_test(veloxcore_rules_collision_test rules/collision_test.cpp)
|
||||
vdm_add_test(veloxcore_rules_match_test rules/match_test.cpp)
|
||||
|
||||
@@ -28,7 +28,14 @@ namespace vdm::testing {
|
||||
|
||||
class TestServer {
|
||||
public:
|
||||
TestServer() {
|
||||
TestServer() : TestServer(1.0) {}
|
||||
|
||||
// loris_seconds overrides the dribble duration slow-loris mode uses (default matches the
|
||||
// no-arg ctor's long-standing 1s). A test that needs curl's stall detector
|
||||
// (CURLOPT_LOW_SPEED_TIME, hardcoded to 30s in download_task.cpp) to actually fire needs a
|
||||
// dribble that outlasts that threshold, not the short one every other test relies on to
|
||||
// keep runtime down.
|
||||
explicit TestServer(double loris_seconds) {
|
||||
const char *script = VDM_TESTSERVER_PY;
|
||||
if (!script || !*script || ::access(script, R_OK) != 0)
|
||||
return;
|
||||
@@ -50,8 +57,9 @@ class TestServer {
|
||||
int devnull = ::open("/dev/null", O_WRONLY);
|
||||
if (devnull >= 0)
|
||||
::dup2(devnull, STDERR_FILENO);
|
||||
std::string loris_str = std::to_string(loris_seconds);
|
||||
::execlp("python3", "python3", script, "--port", "0", "--seed", "9", "--loris-seconds",
|
||||
"1", "--throttle-bps", "131072", static_cast<char *>(nullptr));
|
||||
loris_str.c_str(), "--throttle-bps", "131072", static_cast<char *>(nullptr));
|
||||
::_exit(127);
|
||||
}
|
||||
::close(pipefd[1]);
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
#include "vdm/rules/collision.hpp"
|
||||
|
||||
#include <functional>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#include "vtest.hpp"
|
||||
|
||||
using vdm::rules::CollisionPolicy;
|
||||
using vdm::rules::resolve_collision;
|
||||
|
||||
namespace {
|
||||
std::function<bool(std::string_view)> exists_in(const std::set<std::string> &names) {
|
||||
return [&names](std::string_view s) { return names.count(std::string(s)) > 0; };
|
||||
}
|
||||
} // namespace
|
||||
|
||||
VT_TEST(collision_no_collision_returns_desired) {
|
||||
std::set<std::string> existing = {"other.txt"};
|
||||
VT_CHECK_EQ(resolve_collision("file.txt", exists_in(existing)), std::string("file.txt"));
|
||||
}
|
||||
|
||||
VT_TEST(collision_renames_on_conflict) {
|
||||
std::set<std::string> existing = {"file.txt"};
|
||||
VT_CHECK_EQ(resolve_collision("file.txt", exists_in(existing)), std::string("file (1).txt"));
|
||||
}
|
||||
|
||||
VT_TEST(collision_finds_first_free_slot) {
|
||||
std::set<std::string> existing = {"file.txt", "file (1).txt", "file (2).txt"};
|
||||
VT_CHECK_EQ(resolve_collision("file.txt", exists_in(existing)), std::string("file (3).txt"));
|
||||
}
|
||||
|
||||
VT_TEST(collision_no_extension) {
|
||||
std::set<std::string> existing = {"README"};
|
||||
VT_CHECK_EQ(resolve_collision("README", exists_in(existing)), std::string("README (1)"));
|
||||
}
|
||||
|
||||
VT_TEST(collision_dotfile_treated_as_no_extension) {
|
||||
std::set<std::string> existing = {".gitignore"};
|
||||
VT_CHECK_EQ(resolve_collision(".gitignore", exists_in(existing)),
|
||||
std::string(".gitignore (1)"));
|
||||
}
|
||||
|
||||
VT_TEST(collision_overwrite_policy_ignores_existence) {
|
||||
std::set<std::string> existing = {"file.txt"};
|
||||
VT_CHECK_EQ(resolve_collision("file.txt", exists_in(existing), CollisionPolicy::overwrite),
|
||||
std::string("file.txt"));
|
||||
}
|
||||
|
||||
VT_TEST(collision_gives_up_after_max_attempts_without_fabricating) {
|
||||
auto always_exists = [](std::string_view) { return true; };
|
||||
auto out = resolve_collision("file.txt", always_exists, CollisionPolicy::rename, 3);
|
||||
VT_CHECK_EQ(out, std::string("file (3).txt")); // last attempted, still colliding
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
#include "vdm/rules/filename.hpp"
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "vtest.hpp"
|
||||
|
||||
using vdm::rules::sanitize_filename;
|
||||
|
||||
VT_TEST(filename_passthrough_when_already_clean) {
|
||||
VT_CHECK_EQ(sanitize_filename("report.pdf"), std::string("report.pdf"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_strips_path_separators) {
|
||||
VT_CHECK_EQ(sanitize_filename("a/b\\c.txt"), std::string("abc.txt"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_collapses_dotdot_after_separator_strip) {
|
||||
// "../../etc/passwd" -> separators stripped, then the resulting ".." runs collapse to
|
||||
// a single '.', which strip-trailing-dot then removes entirely.
|
||||
auto out = sanitize_filename("../../etc/passwd");
|
||||
VT_CHECK(out.find("..") == std::string::npos);
|
||||
}
|
||||
|
||||
VT_TEST(filename_strips_control_bytes) {
|
||||
std::string raw = "bad";
|
||||
raw.push_back('\0');
|
||||
raw += "name.txt";
|
||||
auto out = sanitize_filename(raw);
|
||||
VT_CHECK_EQ(out, std::string("badname.txt"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_replaces_ntfs_illegal_chars) {
|
||||
VT_CHECK_EQ(sanitize_filename("a<b>c:d\"e|f?g*h.txt"), std::string("a_b_c_d_e_f_g_h.txt"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_strips_trailing_dot_and_space) {
|
||||
VT_CHECK_EQ(sanitize_filename("name. "), std::string("name"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_empty_falls_back_to_download) {
|
||||
VT_CHECK_EQ(sanitize_filename(""), std::string("download"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_all_stripped_falls_back_to_download) {
|
||||
VT_CHECK_EQ(sanitize_filename("/\\"), std::string("download"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_reserved_device_name_bare) {
|
||||
VT_CHECK_EQ(sanitize_filename("CON"), std::string("CON_"));
|
||||
VT_CHECK_EQ(sanitize_filename("con"), std::string("con_"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_reserved_device_name_with_extension) {
|
||||
VT_CHECK_EQ(sanitize_filename("NUL.txt"), std::string("NUL_.txt"));
|
||||
VT_CHECK_EQ(sanitize_filename("com3.tar.gz"), std::string("com3_.tar.gz"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_reserved_device_name_not_a_false_positive) {
|
||||
// "CONTEST" is not "CON" — must not get mangled.
|
||||
VT_CHECK_EQ(sanitize_filename("CONTEST.txt"), std::string("CONTEST.txt"));
|
||||
VT_CHECK_EQ(sanitize_filename("COM99.txt"), std::string("COM99.txt")); // not COM1-9
|
||||
}
|
||||
|
||||
VT_TEST(filename_truncates_long_name_keeping_extension) {
|
||||
std::string stem(500, 'a');
|
||||
auto out = sanitize_filename(stem + ".txt", 255);
|
||||
VT_CHECK(out.size() <= 255);
|
||||
VT_CHECK(out.ends_with(".txt"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_truncation_is_utf8_safe) {
|
||||
// Each "é" is 2 bytes (C3 A9); a 5-byte budget can fit 2 whole codepoints (4 bytes) but
|
||||
// not a 3rd (needs 6) -- an unguarded byte-length cut at 5 would split the 3rd
|
||||
// codepoint's C3 from its A9, leaving a dangling lead byte.
|
||||
std::string stem;
|
||||
for (int i = 0; i < 20; ++i) stem += "\xC3\xA9";
|
||||
auto out = sanitize_filename(stem, 5);
|
||||
VT_CHECK_EQ(out, std::string("\xC3\xA9\xC3\xA9")); // 2 whole codepoints, 4 bytes
|
||||
}
|
||||
|
||||
VT_TEST(filename_preserves_non_ascii) {
|
||||
VT_CHECK_EQ(sanitize_filename("caf\xC3\xA9.pdf"), std::string("caf\xC3\xA9.pdf"));
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
#include "vdm/rules/match.hpp"
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "vtest.hpp"
|
||||
|
||||
using namespace vdm::rules;
|
||||
|
||||
namespace {
|
||||
|
||||
Rule make_rule(std::string id, std::int64_t priority, RuleMatch m, RuleAction a,
|
||||
bool enabled = true) {
|
||||
Rule r;
|
||||
r.rule_id = std::move(id);
|
||||
r.priority = priority;
|
||||
r.enabled = enabled;
|
||||
r.match = std::move(m);
|
||||
r.action = std::move(a);
|
||||
return r;
|
||||
}
|
||||
|
||||
// -Wmissing-field-initializers (part of -Wextra) flags a designated-initializer list that
|
||||
// skips a member, even one this repo's designated-init style would normally leave
|
||||
// implicit -- these small builders keep the tests below readable without tripping it.
|
||||
RuleAction action_with_category(std::string id) {
|
||||
RuleAction a;
|
||||
a.category_id = std::move(id);
|
||||
return a;
|
||||
}
|
||||
|
||||
MatchInput input_with_extension(std::string ext) {
|
||||
MatchInput in;
|
||||
in.extension = std::move(ext);
|
||||
return in;
|
||||
}
|
||||
|
||||
MatchInput input_with_host(std::string host) {
|
||||
MatchInput in;
|
||||
in.host = std::move(host);
|
||||
return in;
|
||||
}
|
||||
|
||||
MatchInput input_with_size(std::optional<std::uint64_t> size) {
|
||||
MatchInput in;
|
||||
in.size_bytes = size;
|
||||
return in;
|
||||
}
|
||||
|
||||
MatchInput input_with_extension_and_host(std::string ext, std::string host) {
|
||||
MatchInput in;
|
||||
in.extension = std::move(ext);
|
||||
in.host = std::move(host);
|
||||
return in;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// --- glob_match ---------------------------------------------------------------------
|
||||
|
||||
VT_TEST(glob_exact_match) {
|
||||
VT_CHECK(glob_match("example.com", "example.com"));
|
||||
VT_CHECK(!glob_match("example.com", "example.org"));
|
||||
}
|
||||
|
||||
VT_TEST(glob_star_suffix) {
|
||||
VT_CHECK(glob_match("*.example.com", "cdn.example.com"));
|
||||
VT_CHECK(glob_match("*.example.com", "a.b.example.com"));
|
||||
VT_CHECK(!glob_match("*.example.com", "example.com")); // no room for the literal '.'
|
||||
}
|
||||
|
||||
VT_TEST(glob_star_matches_empty) {
|
||||
VT_CHECK(glob_match("file*.zip", "file.zip"));
|
||||
VT_CHECK(glob_match("file*.zip", "file123.zip"));
|
||||
}
|
||||
|
||||
VT_TEST(glob_question_mark) {
|
||||
VT_CHECK(glob_match("file?.txt", "file1.txt"));
|
||||
VT_CHECK(!glob_match("file?.txt", "file.txt"));
|
||||
VT_CHECK(!glob_match("file?.txt", "file12.txt"));
|
||||
}
|
||||
|
||||
VT_TEST(glob_case_insensitive) {
|
||||
VT_CHECK(glob_match("*.EXAMPLE.com", "cdn.example.COM"));
|
||||
}
|
||||
|
||||
VT_TEST(glob_multiple_stars) {
|
||||
VT_CHECK(glob_match("*foo*bar*", "xxfooyybarzz"));
|
||||
VT_CHECK(!glob_match("*foo*bar*", "xxbarzzfooyy")); // order matters
|
||||
}
|
||||
|
||||
VT_TEST(glob_pathological_stars_terminate) {
|
||||
// A pattern of nothing but '*' against a long text must not blow up (bounded work).
|
||||
std::string pattern(50, '*');
|
||||
std::string text(10000, 'x');
|
||||
VT_CHECK(glob_match(pattern, text));
|
||||
}
|
||||
|
||||
// --- match_rules ---------------------------------------------------------------------
|
||||
|
||||
VT_TEST(match_empty_table_yields_nullopt) {
|
||||
VT_CHECK(!match_rules({}, input_with_extension("zip")).has_value());
|
||||
}
|
||||
|
||||
VT_TEST(match_no_clause_matches_everything) {
|
||||
auto rules = {make_rule("r1", 0, RuleMatch{}, action_with_category("default"))};
|
||||
auto r = match_rules(rules, input_with_extension("anything"));
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(*r->category_id, std::string("default"));
|
||||
}
|
||||
|
||||
VT_TEST(match_by_extension_case_insensitive) {
|
||||
RuleMatch m;
|
||||
m.extensions = std::vector<std::string>{"zip", "rar"};
|
||||
auto rules = {make_rule("r1", 0, m, action_with_category("archives"))};
|
||||
VT_CHECK(match_rules(rules, input_with_extension("ZIP")).has_value());
|
||||
VT_CHECK(!match_rules(rules, input_with_extension("txt")).has_value());
|
||||
}
|
||||
|
||||
VT_TEST(match_priority_order_lower_wins) {
|
||||
std::vector<Rule> rules = {
|
||||
make_rule("hi", 10, RuleMatch{}, action_with_category("first")),
|
||||
make_rule("lo", 0, RuleMatch{}, action_with_category("second")),
|
||||
};
|
||||
auto r = match_rules(rules, MatchInput{});
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(*r->category_id, std::string("second")); // priority 0 runs first
|
||||
}
|
||||
|
||||
VT_TEST(match_ties_keep_table_order) {
|
||||
std::vector<Rule> rules = {
|
||||
make_rule("a", 5, RuleMatch{}, action_with_category("first")),
|
||||
make_rule("b", 5, RuleMatch{}, action_with_category("second")),
|
||||
};
|
||||
auto r = match_rules(rules, MatchInput{});
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(*r->category_id, std::string("first"));
|
||||
}
|
||||
|
||||
VT_TEST(match_skips_disabled_rules) {
|
||||
std::vector<Rule> rules = {
|
||||
make_rule("a", 0, RuleMatch{}, action_with_category("disabled"), /*enabled=*/false),
|
||||
make_rule("b", 1, RuleMatch{}, action_with_category("enabled")),
|
||||
};
|
||||
auto r = match_rules(rules, MatchInput{});
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(*r->category_id, std::string("enabled"));
|
||||
}
|
||||
|
||||
VT_TEST(match_host_pattern) {
|
||||
RuleMatch m;
|
||||
m.host_pattern = "*.cdn.example.com";
|
||||
auto rules = {make_rule("r1", 0, m, action_with_category("cdn"))};
|
||||
VT_CHECK(match_rules(rules, input_with_host("a.cdn.example.com")).has_value());
|
||||
VT_CHECK(!match_rules(rules, input_with_host("example.com")).has_value());
|
||||
}
|
||||
|
||||
VT_TEST(match_size_bounds) {
|
||||
RuleMatch m;
|
||||
m.min_size_bytes = 1000;
|
||||
m.max_size_bytes = 2000;
|
||||
auto rules = {make_rule("r1", 0, m, action_with_category("midsize"))};
|
||||
VT_CHECK(match_rules(rules, input_with_size(1500)).has_value());
|
||||
VT_CHECK(!match_rules(rules, input_with_size(500)).has_value());
|
||||
VT_CHECK(!match_rules(rules, input_with_size(5000)).has_value());
|
||||
}
|
||||
|
||||
VT_TEST(match_size_clause_with_unknown_size_does_not_match) {
|
||||
RuleMatch m;
|
||||
m.min_size_bytes = 1000;
|
||||
auto rules = {make_rule("r1", 0, m, action_with_category("big"))};
|
||||
// size_bytes left absent (pre-probe) -- a size clause must not match speculatively.
|
||||
VT_CHECK(!match_rules(rules, MatchInput{}).has_value());
|
||||
}
|
||||
|
||||
VT_TEST(match_all_clauses_must_hold) {
|
||||
RuleMatch m;
|
||||
m.extensions = std::vector<std::string>{"iso"};
|
||||
m.host_pattern = "*.trusted.example";
|
||||
auto rules = {make_rule("r1", 0, m, action_with_category("isos"))};
|
||||
VT_CHECK(match_rules(rules, input_with_extension_and_host("iso", "mirror.trusted.example"))
|
||||
.has_value());
|
||||
// Extension matches but host doesn't -- must not match.
|
||||
VT_CHECK(!match_rules(rules, input_with_extension_and_host("iso", "evil.example"))
|
||||
.has_value());
|
||||
}
|
||||
|
||||
VT_TEST(match_falls_through_to_default_when_nothing_matches) {
|
||||
RuleMatch m;
|
||||
m.extensions = std::vector<std::string>{"exe"};
|
||||
auto rules = {make_rule("r1", 0, m, action_with_category("installers"))};
|
||||
VT_CHECK(!match_rules(rules, input_with_extension("pdf")).has_value());
|
||||
}
|
||||
@@ -2,7 +2,10 @@
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
@@ -20,29 +23,153 @@ TaskId tid(std::uint64_t v) {
|
||||
|
||||
// A test task that reacts to slot targets the way stage 8's download_task will: start
|
||||
// workers up to the target, release them when the target drops. Purely bookkeeping.
|
||||
//
|
||||
// Production's real equivalent (register_task()'s on_target lambda, download_task.cpp)
|
||||
// never calls back in synchronously -- it posts through host.schedule() (engine.cpp), so
|
||||
// apply_slot_target() never runs on the same call stack as whatever budget call triggered
|
||||
// it. wake_one_waiter_locked()'s wakeups, though, mean on_target() here *can* now be
|
||||
// reentered on the same thread (e.g. this task's own release_slot() call, deep in the grow
|
||||
// loop below, can cascade into waking a *different* task whose own release, in turn,
|
||||
// cascades back to this one) -- a plain std::mutex would self-deadlock on that, the way
|
||||
// FakeTask::mu almost did until this was written. A recursive_mutex plus coalescing the
|
||||
// reentrant call's target into `pending` (processed by the outer call's loop once its
|
||||
// current pass finishes) keeps this a faithful, deterministic stand-in without actually
|
||||
// needing real threads: no callback ever runs nested inside a still-running instance of
|
||||
// itself, and a burst of reentrant retargeting converges to the latest value instead of
|
||||
// each one doing its own redundant grow/shrink pass.
|
||||
struct FakeTask {
|
||||
SegmentBudget *budget = nullptr;
|
||||
TaskId id{};
|
||||
std::mutex mu;
|
||||
std::recursive_mutex mu;
|
||||
std::uint32_t workers = 0;
|
||||
std::uint32_t target = 0;
|
||||
bool running = false;
|
||||
std::optional<std::uint32_t> pending;
|
||||
|
||||
FakeTask() = default;
|
||||
FakeTask(SegmentBudget *b, TaskId i) : budget(b), id(i) {}
|
||||
|
||||
void on_target(std::uint32_t t) {
|
||||
std::lock_guard lk(mu);
|
||||
target = t;
|
||||
while (workers < target) {
|
||||
if (!budget->confirm_slot(id))
|
||||
if (running) {
|
||||
pending = t; // reentrant call on this thread -- the running instance picks it up
|
||||
return;
|
||||
}
|
||||
running = true;
|
||||
std::uint32_t want = t;
|
||||
for (;;) {
|
||||
target = want;
|
||||
while (workers < target) {
|
||||
if (!budget->confirm_slot(id))
|
||||
break;
|
||||
++workers;
|
||||
}
|
||||
// over target -> yield the excess immediately (a real task waits for a
|
||||
// boundary)
|
||||
while (workers > target) {
|
||||
budget->release_slot(id);
|
||||
--workers;
|
||||
}
|
||||
if (!pending)
|
||||
break;
|
||||
++workers;
|
||||
want = *pending;
|
||||
pending.reset();
|
||||
}
|
||||
// over target -> yield the excess immediately (a real task waits for a boundary)
|
||||
while (workers > target) {
|
||||
budget->release_slot(id);
|
||||
--workers;
|
||||
running = false;
|
||||
}
|
||||
std::uint32_t held() {
|
||||
std::lock_guard lk(mu);
|
||||
return workers;
|
||||
}
|
||||
};
|
||||
|
||||
// Mirrors Engine's real timer thread (engine.cpp: timer_loop) and how register_task()'s
|
||||
// on_target lambda actually reaches a task (download_task.cpp: host.schedule(), never a
|
||||
// direct call). One dedicated thread drains queued closures one at a time -- the
|
||||
// serialization that makes production immune to the cross-task deadlock risk FakeTask's
|
||||
// synchronous, whatever-thread-triggered-it delivery has under heavy *concurrent* driving
|
||||
// (budget_concurrent_confirm_release_stays_consistent, below, is the one test that
|
||||
// actually exercises this: multiple threads calling set_want() for *different* tasks at
|
||||
// once, each able to cascade into a wake for another task's callback -- two such cascades
|
||||
// landing on two different FakeTask mutexes in opposite orders on two different threads is
|
||||
// a real AB-BA deadlock a recursive_mutex alone doesn't prevent, since that only guards a
|
||||
// single thread against re-entering itself). Every other test in this file drives the
|
||||
// budget from one thread at a time, where that risk can't arise, so plain FakeTask is
|
||||
// still the simpler, sufficient double there.
|
||||
class TestTimer {
|
||||
public:
|
||||
TestTimer() {
|
||||
worker_ = std::jthread([this](std::stop_token st) { run(st); });
|
||||
}
|
||||
~TestTimer() {
|
||||
worker_.request_stop();
|
||||
cv_.notify_all();
|
||||
}
|
||||
void post(std::function<void()> fn) {
|
||||
{
|
||||
std::lock_guard lk(mu_);
|
||||
queue_.push_back(std::move(fn));
|
||||
}
|
||||
cv_.notify_one();
|
||||
}
|
||||
// Blocks until the queue is empty and nothing is mid-run -- what a test needs before
|
||||
// asserting on state this timer's closures mutate.
|
||||
void drain() {
|
||||
std::unique_lock lk(mu_);
|
||||
cv_done_.wait(lk, [&] { return queue_.empty() && !running_; });
|
||||
}
|
||||
|
||||
private:
|
||||
void run(std::stop_token st) {
|
||||
std::unique_lock lk(mu_);
|
||||
while (true) {
|
||||
cv_.wait(lk, st, [&] { return !queue_.empty(); });
|
||||
if (st.stop_requested())
|
||||
return;
|
||||
auto fn = std::move(queue_.front());
|
||||
queue_.erase(queue_.begin());
|
||||
running_ = true;
|
||||
lk.unlock();
|
||||
fn();
|
||||
lk.lock();
|
||||
running_ = false;
|
||||
if (queue_.empty())
|
||||
cv_done_.notify_all();
|
||||
}
|
||||
}
|
||||
std::mutex mu_;
|
||||
std::condition_variable_any cv_; // _any: wait() below takes a stop_token predicate
|
||||
std::condition_variable cv_done_;
|
||||
std::vector<std::function<void()>> queue_;
|
||||
bool running_ = false;
|
||||
std::jthread worker_;
|
||||
};
|
||||
|
||||
// Same grow/shrink logic as FakeTask, but on_target() only ever posts through a TestTimer
|
||||
// instead of running synchronously -- see the comment above TestTimer for why that's the
|
||||
// faithful model under concurrent driving.
|
||||
struct AsyncFakeTask {
|
||||
SegmentBudget *budget = nullptr;
|
||||
TestTimer *timer = nullptr;
|
||||
TaskId id{};
|
||||
std::mutex mu; // only the timer thread ever touches workers/target -- plain suffices
|
||||
std::uint32_t workers = 0;
|
||||
std::uint32_t target = 0;
|
||||
|
||||
void on_target(std::uint32_t t) {
|
||||
timer->post([this, t] {
|
||||
std::lock_guard lk(mu);
|
||||
target = t;
|
||||
while (workers < target) {
|
||||
if (!budget->confirm_slot(id))
|
||||
break;
|
||||
++workers;
|
||||
}
|
||||
while (workers > target) {
|
||||
budget->release_slot(id);
|
||||
--workers;
|
||||
}
|
||||
});
|
||||
}
|
||||
std::uint32_t held() {
|
||||
std::lock_guard lk(mu);
|
||||
@@ -235,12 +362,14 @@ VT_TEST(budget_on_changed_fires_on_starved_edge) {
|
||||
VT_TEST(budget_concurrent_confirm_release_stays_consistent) {
|
||||
SegmentBudget b({.max_active_segments = 16});
|
||||
constexpr int kTasks = 6;
|
||||
std::vector<std::unique_ptr<FakeTask>> ts;
|
||||
TestTimer timer;
|
||||
std::vector<std::unique_ptr<AsyncFakeTask>> ts;
|
||||
for (int i = 0; i < kTasks; ++i) {
|
||||
ts.push_back(std::make_unique<FakeTask>());
|
||||
ts.push_back(std::make_unique<AsyncFakeTask>());
|
||||
ts.back()->budget = &b;
|
||||
ts.back()->timer = &timer;
|
||||
ts.back()->id = tid(i + 1);
|
||||
FakeTask *ft = ts.back().get();
|
||||
AsyncFakeTask *ft = ts.back().get();
|
||||
b.register_task(ft->id, {.host = "h", .per_task_cap = 6, .resumable = true},
|
||||
[ft](std::uint32_t n) { ft->on_target(n); });
|
||||
}
|
||||
@@ -254,6 +383,7 @@ VT_TEST(budget_concurrent_confirm_release_stays_consistent) {
|
||||
drivers.clear(); // join
|
||||
for (auto &ft : ts)
|
||||
b.set_want(ft->id, 0);
|
||||
timer.drain(); // let every queued on_target actually run before asserting
|
||||
|
||||
// With everyone wanting nothing, the budget must be fully released.
|
||||
VT_CHECK_EQ(b.budget().active, 0u);
|
||||
@@ -262,3 +392,123 @@ VT_TEST(budget_concurrent_confirm_release_stays_consistent) {
|
||||
sum += b.segments_active(ft->id);
|
||||
VT_CHECK_EQ(sum, 0u);
|
||||
}
|
||||
|
||||
// --- wait-list wakeup: tools/bench heap-profile / load found a real task time out
|
||||
// waiting on a slot its own target already said it should have (core/docs/m7-baseline.md,
|
||||
// docs/adr/0012). Root cause: confirm_slot()'s engine-wide cap check (needed so active_
|
||||
// never exceeds max_active_segments -- a real over-admission bug, not just this liveness
|
||||
// gap) can deny a task whose target is already correct, when a *different* task is
|
||||
// legitimately still holding more than its own just-lowered target (yield is deferred to
|
||||
// a segment boundary, ADR 0011 A1). Nothing in the plain target-changed callback
|
||||
// mechanism ever revisits a task whose target didn't change -- it was already right. ---
|
||||
|
||||
namespace {
|
||||
|
||||
// Unlike FakeTask above, on_target() here only enqueues -- it never calls back into the
|
||||
// budget synchronously. This matches production exactly: register_task()'s on_target
|
||||
// lambda (download_task.cpp) posts through host.schedule() (engine.cpp), so
|
||||
// apply_slot_target() never runs on the same call stack as whatever budget call triggered
|
||||
// it. The test drives delivery explicitly (deliver_one()) instead of a background thread
|
||||
// so the race this test exists to force -- confirm_slot() denied before the task that's
|
||||
// over its target has processed its own shrink -- is deterministic, not a timing gamble.
|
||||
struct QueuedTask {
|
||||
SegmentBudget *budget = nullptr;
|
||||
TaskId id{};
|
||||
std::uint32_t workers = 0;
|
||||
std::uint32_t target = 0;
|
||||
std::vector<std::uint32_t> pending;
|
||||
|
||||
QueuedTask(SegmentBudget *b, TaskId i) : budget(b), id(i) {}
|
||||
|
||||
void on_target(std::uint32_t n) { pending.push_back(n); }
|
||||
|
||||
// Delivers the oldest queued target, applying it the way a real task's
|
||||
// apply_slot_target()/fill_slots_locked() would: try to grow to it (confirm_slot()
|
||||
// may deny), or shed down to it. Returns false (nothing to deliver) if the queue was
|
||||
// empty -- the condition VT_REQUIRE checks to prove a wakeup was actually queued.
|
||||
bool deliver_one() {
|
||||
if (pending.empty())
|
||||
return false;
|
||||
target = pending.front();
|
||||
pending.erase(pending.begin());
|
||||
while (workers < target) {
|
||||
if (!budget->confirm_slot(id))
|
||||
break;
|
||||
++workers;
|
||||
}
|
||||
while (workers > target) {
|
||||
budget->release_slot(id);
|
||||
--workers;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
VT_TEST(budget_release_wakes_a_denied_waiter) {
|
||||
// Sanity baseline: max_active=1, A (higher priority) holds it, B wants one too and is
|
||||
// fairly denied -- its target stays 0 while A outranks it and still wants its slot.
|
||||
// Once A stops wanting one, B's target rises and B is woken via the plain
|
||||
// target-changed path -- no wait-list needed for this simple case. The harder case
|
||||
// below is what actually needs it.
|
||||
SegmentBudget b({.max_active_segments = 1});
|
||||
QueuedTask A{&b, tid(1)}, B{&b, tid(2)};
|
||||
b.register_task(A.id, {.host = "h", .per_task_cap = 1, .resumable = true},
|
||||
[&](std::uint32_t n) { A.on_target(n); });
|
||||
b.set_task_order(std::vector<TaskId>{A.id, B.id});
|
||||
b.set_want(A.id, 1);
|
||||
VT_REQUIRE(A.deliver_one());
|
||||
VT_CHECK_EQ(A.workers, 1u);
|
||||
|
||||
// B registers and wants one too, but with A (higher priority) already holding the
|
||||
// only slot and still wanting it, B's fairly computed target stays 0 -- unchanged
|
||||
// from its just-registered value, so no callback is queued for it yet.
|
||||
b.register_task(B.id, {.host = "h", .per_task_cap = 1, .resumable = true},
|
||||
[&](std::uint32_t n) { B.on_target(n); });
|
||||
b.set_want(B.id, 1);
|
||||
VT_CHECK(!B.deliver_one());
|
||||
VT_CHECK_EQ(B.workers, 0u);
|
||||
|
||||
b.set_want(A.id, 0); // A is done wanting a slot
|
||||
VT_REQUIRE(A.deliver_one()); // A's target dropped to 0 -- sheds its held slot
|
||||
VT_CHECK_EQ(A.workers, 0u);
|
||||
VT_REQUIRE(B.deliver_one()); // B's target rose to 1 -- the plain target-changed path
|
||||
VT_CHECK_EQ(B.workers, 1u); // B took the freed slot
|
||||
}
|
||||
|
||||
VT_TEST(budget_wait_list_wakes_a_task_whose_target_never_changed) {
|
||||
// The real gap. max_active=2. Y alone, holds both (target=2). X arrives wanting 1:
|
||||
// this recompute correctly drops Y's target to 1 (giving X its guaranteed slot) and
|
||||
// raises X's target to 1 -- both real target changes, both queued. Deliver X's
|
||||
// *first*: X's target says grow, but Y still physically holds 2 (hasn't processed
|
||||
// its own shrink yet) -- confirm_slot() must deny X here (active_ == max_active_),
|
||||
// which is the correctness fix (over-admission is the real RSS bug). Then Y
|
||||
// processes its shrink and actually releases. X's target never changes again -- it
|
||||
// was already correctly 1 -- so nothing in the plain mechanism ever revisits X.
|
||||
SegmentBudget b({.max_active_segments = 2});
|
||||
QueuedTask Y{&b, tid(1)}, X{&b, tid(2)};
|
||||
b.register_task(Y.id, {.host = "h", .per_task_cap = 2, .resumable = true},
|
||||
[&](std::uint32_t n) { Y.on_target(n); });
|
||||
b.set_want(Y.id, 2);
|
||||
VT_REQUIRE(Y.deliver_one());
|
||||
VT_CHECK_EQ(Y.workers, 2u);
|
||||
|
||||
b.register_task(X.id, {.host = "h", .per_task_cap = 1, .resumable = true},
|
||||
[&](std::uint32_t n) { X.on_target(n); });
|
||||
b.set_task_order(std::vector<TaskId>{Y.id, X.id});
|
||||
b.set_want(X.id, 1);
|
||||
|
||||
VT_REQUIRE(X.deliver_one());
|
||||
VT_CHECK_EQ(X.workers, 0u); // denied: active_ == max_active_, even though X's target is 1
|
||||
|
||||
VT_REQUIRE(Y.deliver_one());
|
||||
VT_CHECK_EQ(Y.workers, 1u); // Y actually releases its excess now
|
||||
|
||||
// The bug: without a wait-list, X.pending is empty here -- nothing was ever queued
|
||||
// for it, because X's target never changed again. X would wait forever despite its
|
||||
// target correctly saying it should hold a slot.
|
||||
VT_REQUIRE(X.deliver_one());
|
||||
VT_CHECK_EQ(X.workers, 1u);
|
||||
VT_CHECK_EQ(b.budget().active, 2u);
|
||||
}
|
||||
|
||||
@@ -62,6 +62,7 @@ struct Recorder {
|
||||
std::lock_guard lk(mu);
|
||||
states.push_back(to);
|
||||
};
|
||||
c.on_decision_needed = [this](const DecisionRequest &) { decision_calls.fetch_add(1); };
|
||||
c.on_finished = [this](Result<DownloadOutcome> r) {
|
||||
if (!fired.exchange(true))
|
||||
done.set_value(std::move(r));
|
||||
@@ -123,6 +124,31 @@ std::string server_sha(TestServer &srv, const std::string &mode, const std::stri
|
||||
return out.substr(open + 1, close - open - 1);
|
||||
}
|
||||
|
||||
// Small, deliberately identical extraction to server_sha's: GET /<mode>/sign/<size>?ttl=N
|
||||
// and pull the "url" field's value out of the {"url":..., "exp":...} JSON body.
|
||||
std::string sign_url(TestServer &srv, const std::string &mode, const std::string &size,
|
||||
int ttl_seconds) {
|
||||
std::string url =
|
||||
srv.url("/" + mode + "/sign/" + size + "?ttl=" + std::to_string(ttl_seconds));
|
||||
std::string cmd = "curl -s '" + url + "'";
|
||||
std::string out;
|
||||
if (FILE *f = ::popen(cmd.c_str(), "r")) {
|
||||
char buf[1024];
|
||||
while (std::fgets(buf, sizeof buf, f))
|
||||
out += buf;
|
||||
::pclose(f);
|
||||
}
|
||||
auto q = out.find("\"url\"");
|
||||
if (q == std::string::npos)
|
||||
return {};
|
||||
auto colon = out.find(':', q);
|
||||
auto open = out.find('"', colon);
|
||||
auto close = out.find('"', open + 1);
|
||||
if (open == std::string::npos || close == std::string::npos)
|
||||
return {};
|
||||
return out.substr(open + 1, close - open - 1);
|
||||
}
|
||||
|
||||
DownloadSpec spec_for(TestServer &srv, const std::string &urlpath, const std::string &save) {
|
||||
DownloadSpec s;
|
||||
s.url = srv.url(urlpath);
|
||||
@@ -326,3 +352,323 @@ VT_TEST(engine_401_then_provide_auth_completes) {
|
||||
VT_CHECK(rec.auth_calls.load() >= 1);
|
||||
VT_CHECK_EQ(file_size(td.file("au.bin")), 1u * 1024 * 1024);
|
||||
}
|
||||
|
||||
VT_TEST(engine_401_digest_then_provide_auth_completes) {
|
||||
// Same shape as engine_401_then_provide_auth_completes, but the challenge is HTTP
|
||||
// Digest (qop=auth) rather than Basic. provide_auth() doesn't know or care which --
|
||||
// http_client.cpp always asks libcurl for CURLAUTH_ANY (net::AuthScheme::any) and lets
|
||||
// curl negotiate against whatever WWW-Authenticate the server actually sent -- so this
|
||||
// exists purely to prove that's true end-to-end, not just at the unit level.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
DownloadHandle h;
|
||||
auto cbs = rec.cbs(&h, "test", "test");
|
||||
h = eng.start(spec_for(srv, "/401-digest/file/1M", td.file("dg.bin")), std::move(cbs));
|
||||
rec.arm(h);
|
||||
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK(rec.auth_calls.load() >= 1);
|
||||
VT_CHECK_EQ(file_size(td.file("dg.bin")), 1u * 1024 * 1024);
|
||||
auto got = hash_file(td.file("dg.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "401-digest", "1M"));
|
||||
}
|
||||
|
||||
// --- hostile-mode matrix: the four where a bug is silent corruption, not a visible
|
||||
// failure (docs/04 §5 "ask, never silently corrupt" / §7's failure-policy table). ---
|
||||
|
||||
VT_TEST(engine_etag_changes_asks_instead_of_splicing) {
|
||||
// A server that revalidates with a different ETag on every response fails an If-Range
|
||||
// on any retry or resume. That must surface as "ask the user" (server_file_changed),
|
||||
// never as a silent restart-from-offset-0 spliced onto bytes already on disk.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/throttled+etag-changes/file/2M", td.file("ec.bin")),
|
||||
rec.cbs());
|
||||
|
||||
// Get real progress on at least one segment before pausing, so resume's If-Range (only
|
||||
// sent once a segment has completed > 0) actually fires.
|
||||
for (int i = 0; i < 300 && h.progress().downloaded < 64u * 1024; ++i)
|
||||
std::this_thread::sleep_for(10ms);
|
||||
VT_REQUIRE(h.progress().downloaded >= 64u * 1024);
|
||||
h.pause();
|
||||
for (int i = 0; i < 200 && h.state() != EngineState::paused; ++i)
|
||||
std::this_thread::sleep_for(20ms);
|
||||
VT_REQUIRE(h.state() == EngineState::paused);
|
||||
h.resume();
|
||||
|
||||
for (int i = 0; i < 300 && rec.decision_calls.load() == 0; ++i)
|
||||
std::this_thread::sleep_for(20ms);
|
||||
VT_REQUIRE(rec.decision_calls.load() >= 1);
|
||||
VT_CHECK_EQ(h.state(), EngineState::paused);
|
||||
|
||||
h.decide(Decision::restart);
|
||||
auto r = rec.wait(90s);
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(file_size(td.file("ec.bin")), 2u * 1024 * 1024);
|
||||
auto got = hash_file(td.file("ec.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "throttled+etag-changes", "2M"));
|
||||
}
|
||||
|
||||
VT_TEST(engine_416_mid_download_asks_instead_of_exhausting_retries) {
|
||||
// 416-always 416s every ranged request, including the probe's own -- a live probe
|
||||
// correctly concludes "not resumable" and a plain-GET download never touches Range
|
||||
// (that path is the same shape as engine_non_resumable_single_segment). The failure
|
||||
// mode docs/04 means -- a server that *was* proven resumable dropping Range support
|
||||
// mid-download -- needs a worker to actually send Range against it, so force the
|
||||
// resumable, multi-segment assumption directly via probe_hint.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
|
||||
net::ProbeResult hint;
|
||||
hint.total_size = 256u * 1024;
|
||||
hint.last_modified = "Wed, 01 Jan 2025 00:00:00 GMT";
|
||||
hint.accept_ranges = true;
|
||||
hint.resumable = true;
|
||||
auto s = spec_for(srv, "/416-always/file/256K", td.file("rb.bin"));
|
||||
s.probe_hint = hint;
|
||||
s.segments = 2;
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
|
||||
for (int i = 0; i < 300 && rec.decision_calls.load() == 0; ++i)
|
||||
std::this_thread::sleep_for(20ms);
|
||||
VT_REQUIRE(rec.decision_calls.load() >= 1);
|
||||
VT_CHECK_EQ(h.state(), EngineState::paused);
|
||||
|
||||
// 416-always never recovers -- re-probing would just 416 again -- so the only sound
|
||||
// resolution is to stop, honestly, rather than retry the stale range until exhaustion.
|
||||
h.decide(Decision::abort);
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::range_not_satisfiable);
|
||||
VT_CHECK_EQ(::access(td.file("rb.bin").c_str(), F_OK), -1); // never declared complete
|
||||
}
|
||||
|
||||
VT_TEST(engine_lies_about_accept_ranges_demotes_without_asking) {
|
||||
// Ranges are always ignored (a plain 200, full body) but ETag/Last-Modified are stable
|
||||
// and honest -- unlike etag-changes, this is provably the *same* file, just a Range-
|
||||
// blind connection. docs/04 §7: demote to 1 segment and continue, automatically, no
|
||||
// user round-trip. As with 416-always, a live probe already gets this right up front
|
||||
// (proven non-resumable), so probe_hint forces the interesting mid-download case.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
|
||||
std::string want = server_sha(srv, "lies-about-accept-ranges", "512K");
|
||||
VT_REQUIRE(!want.empty());
|
||||
net::ProbeResult hint;
|
||||
hint.total_size = 512u * 1024;
|
||||
hint.last_modified = "Wed, 01 Jan 2025 00:00:00 GMT"; // testserver sends this verbatim
|
||||
hint.accept_ranges = true;
|
||||
hint.resumable = true;
|
||||
auto s = spec_for(srv, "/lies-about-accept-ranges/file/512K", td.file("lar.bin"));
|
||||
s.probe_hint = hint;
|
||||
s.segments = 4;
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
|
||||
auto r = rec.wait(60s);
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(rec.decision_calls.load(), 0); // demoted automatically, not asked
|
||||
VT_CHECK_EQ(file_size(td.file("lar.bin")), 512u * 1024);
|
||||
auto got = hash_file(td.file("lar.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), want);
|
||||
}
|
||||
|
||||
VT_TEST(engine_content_length_mismatch_fails_honestly) {
|
||||
// Content-Length promises the true size but the connection always closes short of it.
|
||||
// There is no recovery (unlike flaky-reset, this never "heals" on a later attempt), so
|
||||
// the segment's remaining range shrinks every retry until it stalls at zero progress.
|
||||
// The only correct outcome is a real, visible failure -- never a rename to save_path
|
||||
// built from a file that is quietly missing bytes.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto s = spec_for(srv, "/content-length-mismatch/file/16K", td.file("clm.bin"));
|
||||
s.segments = 1;
|
||||
s.max_retries = 4;
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
auto r = rec.wait(60s);
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::max_retries_exhausted);
|
||||
VT_CHECK(rec.saw(EngineState::failed));
|
||||
VT_CHECK_EQ(::access(td.file("clm.bin").c_str(), F_OK), -1); // never renamed into place
|
||||
}
|
||||
|
||||
// --- remaining hostile-mode matrix (tools/testserver/README.md's mode table). ---
|
||||
|
||||
VT_TEST(engine_expiring_signed_url_recovers_via_refresh_url) {
|
||||
// A signed URL past its ttl 403s (tools/testserver's own JSON body distinguishes
|
||||
// "expired" from "bad signature", but core never parses response bodies -- CLAUDE.md
|
||||
// §3 -- so both just read as a 403). The one automatic referrer retry (see
|
||||
// engine_403_without_referer_retries_with_origin, below) can't fix an expired
|
||||
// signature, so the second 403 asks -- via the same auto_pause_locked(..., false,
|
||||
// true) "ask, don't just fail" path as wrong_status/range_bad -- rather than
|
||||
// terminally failing outright, specifically so DownloadHandle::refresh_url() (its own
|
||||
// contract: works "on a live or paused task", never on a terminal one) stays usable:
|
||||
// the README pairs this mode with exactly that recovery.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
|
||||
std::string expired = sign_url(srv, "expiring-signed-url", "64K", /*ttl=*/1);
|
||||
VT_REQUIRE(!expired.empty());
|
||||
std::this_thread::sleep_for(1500ms); // let the ttl actually pass before the first request
|
||||
|
||||
DownloadSpec s;
|
||||
s.url = expired;
|
||||
s.save_path = td.file("exp.bin");
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
|
||||
for (int i = 0; i < 300 && rec.decision_calls.load() == 0; ++i)
|
||||
std::this_thread::sleep_for(20ms);
|
||||
VT_REQUIRE(rec.decision_calls.load() >= 1);
|
||||
VT_CHECK_EQ(h.state(), EngineState::paused);
|
||||
|
||||
std::string fresh = sign_url(srv, "expiring-signed-url", "64K", /*ttl=*/60);
|
||||
VT_REQUIRE(!fresh.empty());
|
||||
h.refresh_url(fresh);
|
||||
|
||||
auto r = rec.wait(60s);
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(file_size(td.file("exp.bin")), 64u * 1024);
|
||||
auto got = hash_file(td.file("exp.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "expiring-signed-url", "64K"));
|
||||
}
|
||||
|
||||
VT_TEST(engine_403_without_referer_retries_with_origin) {
|
||||
// docs/04 §7: "403 after redirect: retry once with the original referrer -- many CDNs
|
||||
// require it." No spec.referrer is set here (the common case for anything not
|
||||
// initiated from a browser page, e.g. `velox add <url>`), so the first attempt 403s;
|
||||
// the engine's own retry supplies the download URL's own origin as Referer, which
|
||||
// this mode accepts, and the download completes with no decision ever asked.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/403-without-referer/file/128K", td.file("ref.bin")),
|
||||
rec.cbs());
|
||||
auto r = rec.wait(30s);
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(rec.decision_calls.load(), 0); // recovered automatically, not asked
|
||||
VT_CHECK_EQ(file_size(td.file("ref.bin")), 128u * 1024);
|
||||
auto got = hash_file(td.file("ref.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "403-without-referer", "128K"));
|
||||
}
|
||||
|
||||
VT_TEST(engine_redirect_chain_follows_to_completion) {
|
||||
// 5 hops (tools/testserver's own --redirect-depth default) of a plain 302, query
|
||||
// string preserved across each. No CORE-side logic needed for this one -- libcurl's
|
||||
// own CURLOPT_FOLLOWLOCATION (RequestOptions::follow_redirects, already on) and
|
||||
// CURLOPT_MAXREDIRS (default 20, well over 5) do the whole thing -- this is here as
|
||||
// the end-to-end check that they're actually wired through both the probe and every
|
||||
// segment worker's own request, not just one of the two.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/redirect-chain/file/1M", td.file("rc.bin")), rec.cbs());
|
||||
auto r = rec.wait(30s);
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(file_size(td.file("rc.bin")), 1u * 1024 * 1024);
|
||||
auto got = hash_file(td.file("rc.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "redirect-chain", "1M"));
|
||||
}
|
||||
|
||||
VT_TEST(engine_slow_loris_stall_timeout_fires) {
|
||||
// Status line, headers, and body dribbled out one byte at a time for --loris-seconds,
|
||||
// then (if the dribble hasn't already been cut off) normal streaming -- a connection
|
||||
// that's technically alive (bytes ARE arriving, just far too slowly) but must not be
|
||||
// allowed to hang the task forever. http_client.cpp sets CURLOPT_LOW_SPEED_LIMIT/_TIME
|
||||
// (RequestOptions::low_speed_bytes_per_sec/low_speed_secs, hardcoded in
|
||||
// download_task.cpp to 1024 B/s for 30s) for exactly this.
|
||||
//
|
||||
// Every other test in this file uses TestServer's default 1s loris dribble to keep
|
||||
// runtime down, but 1s is far shorter than curl's 30s low_speed_time: a 1s trickle
|
||||
// followed by full-speed streaming never accumulates 30 CONSECUTIVE seconds under the
|
||||
// floor, so curl would never actually abort it -- the download would just complete
|
||||
// slightly late, which would make this test pass for the wrong reason (or not exercise
|
||||
// the stall timeout at all). Explicitly ask for a dribble that outlasts the 30s
|
||||
// threshold so the stall timeout is the thing actually observed firing, not assumed.
|
||||
TestServer srv(40.0);
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto s = spec_for(srv, "/slow-loris/file/64K", td.file("sl.bin"));
|
||||
s.segments = 1;
|
||||
s.max_retries = 1;
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
auto r = rec.wait(60s); // stall timeout fires ~30s in; must resolve, not hang to 60s
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK(is_retryable(r.error().code) || r.error().code == Error::max_retries_exhausted);
|
||||
}
|
||||
|
||||
VT_TEST(engine_chunked_no_length_completes_single_segment) {
|
||||
// No Content-Length anywhere (HEAD gets none either, since it's the same handler path)
|
||||
// -- the probe can't know total_size or prove resumability, so this should take the
|
||||
// exact same "unknown size, one plain-GET segment" path as engine_non_resumable_single_
|
||||
// segment, just arriving there via a chunked body instead of a server that plainly
|
||||
// refuses Range. No core-side work needed if that demotion is already size-agnostic;
|
||||
// this is here to prove it, since every other test's server tells the probe the size
|
||||
// up front.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/chunked-no-length/file/2M", td.file("ch.bin")),
|
||||
rec.cbs());
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(rec.decision_calls.load(), 0);
|
||||
VT_CHECK_EQ(file_size(td.file("ch.bin")), 2u * 1024 * 1024);
|
||||
auto got = hash_file(td.file("ch.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "chunked-no-length", "2M"));
|
||||
}
|
||||
|
||||
// --- DAEMON-reported bug: Progress.speed_bps reads 0 for the whole life of a live
|
||||
// download while downloaded bytes visibly advance. DAEMON reads progress by polling
|
||||
// DownloadHandle::progress() (engine_port_core.hpp), not the on_progress push callback --
|
||||
// this exercises exactly that path. ---
|
||||
|
||||
VT_TEST(engine_polled_progress_reports_nonzero_speed) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/throttled/file/4M", td.file("sp.bin")), rec.cbs());
|
||||
|
||||
// Give it real, sustained progress: the speed estimate only updates on a >=0.5s
|
||||
// sample window (seg_data), so a snapshot taken too early would legitimately read 0
|
||||
// even with the bug fixed. Poll until downloaded has clearly advanced twice over.
|
||||
std::uint64_t speed = 0;
|
||||
for (int i = 0; i < 400 && speed == 0; ++i) {
|
||||
std::this_thread::sleep_for(20ms);
|
||||
auto p = h.progress();
|
||||
if (p.downloaded >= 256u * 1024)
|
||||
speed = p.speed_bps;
|
||||
}
|
||||
VT_CHECK(speed > 0);
|
||||
|
||||
h.cancel(/*discard_partial=*/true);
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(!r.has_value());
|
||||
}
|
||||
|
||||
+16
-2
@@ -35,6 +35,10 @@ add_library(veloxd_store STATIC
|
||||
src/store/pairings.cpp
|
||||
src/store/settings.cpp
|
||||
src/store/tasks.cpp
|
||||
src/store/categories.cpp
|
||||
src/store/queues.cpp
|
||||
src/store/rules.cpp
|
||||
src/store/segments.cpp
|
||||
${_mig_hdr}
|
||||
)
|
||||
add_library(velox::daemon_store ALIAS veloxd_store)
|
||||
@@ -65,12 +69,14 @@ add_library(velox::daemon_sched ALIAS veloxd_sched)
|
||||
target_include_directories(veloxd_sched PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src)
|
||||
target_compile_features(veloxd_sched PUBLIC cxx_std_23)
|
||||
target_compile_options(veloxd_sched PRIVATE -Wall -Wextra -Wpedantic -Werror)
|
||||
target_link_libraries(veloxd_sched PUBLIC velox::proto velox::core veloxd_store nlohmann_json::nlohmann_json)
|
||||
target_link_libraries(veloxd_sched PUBLIC velox::proto velox::core veloxd_store veloxd_rpc nlohmann_json::nlohmann_json)
|
||||
|
||||
# --- veloxd_rpc — the RPC transports + dispatcher ------------------------------------
|
||||
add_library(veloxd_rpc STATIC
|
||||
src/rpc/runtime_dir.cpp
|
||||
src/rpc/single_instance.cpp
|
||||
src/rpc/event_loop.cpp
|
||||
src/rpc/event_hub.cpp
|
||||
src/rpc/uds_server.cpp
|
||||
src/rpc/ws_frame.cpp
|
||||
src/rpc/ws_handshake.cpp
|
||||
@@ -83,8 +89,16 @@ add_library(velox::daemon_rpc ALIAS veloxd_rpc)
|
||||
target_include_directories(veloxd_rpc PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src)
|
||||
target_compile_features(veloxd_rpc PUBLIC cxx_std_23)
|
||||
target_compile_options(veloxd_rpc PRIVATE -Wall -Wextra -Wpedantic -Werror)
|
||||
# velox::core: dispatcher.hpp includes sched/scheduler.hpp for the Scheduler* it drives
|
||||
# download.pause/resume/start/cancel and queue.start/stop through (D4b), which pulls in
|
||||
# core/include's vdm/*.hpp. Interface-only from here (no .cpp in this library calls into
|
||||
# CORE directly) — the actual Scheduler symbols resolve at the veloxd executable's link
|
||||
# step (veloxd links both veloxd_rpc and veloxd_sched), not here, so this does not create
|
||||
# the veloxd_rpc <-> veloxd_sched cycle that linking veloxd_sched itself would (veloxd_sched
|
||||
# already links veloxd_rpc, for EventHub).
|
||||
target_link_libraries(veloxd_rpc
|
||||
PUBLIC velox::proto veloxd_store veloxd_fs nlohmann_json::nlohmann_json Threads::Threads
|
||||
PUBLIC velox::proto velox::core veloxd_store veloxd_fs nlohmann_json::nlohmann_json
|
||||
Threads::Threads
|
||||
)
|
||||
|
||||
# --- veloxd — the daemon binary -------------------------------------------------------
|
||||
|
||||
@@ -5,9 +5,25 @@ close. Kept here (not buried in commit messages) so the next pass can see them a
|
||||
|
||||
| # | What | Where | Why deferred | Closes when |
|
||||
|---|---|---|---|---|
|
||||
| ~~D7~~ | **Closed — `capture.offer` is real.** Applies `capture.enabled`/`excludedHosts`/`monitoredExtensions`/`monitoredMimeTypes`/`minSizeBytes` from settings, then the rules table (`store::Rules` + CORE's `vdm::rules::match_rules`/`glob_match` — DAEMON only converts its own stored `proto::Rule` JSON into CORE's plain `vdm::rules::Rule` vocabulary, per that header's own layering note), resolves the category folder (a rule's explicit `categoryId`/`saveDir`, else `store::Categories::guess_by_extension` — the same extension-guess `download.probe`'s `suggestedCategoryId` already used, now shared instead of duplicated), dedupes against active (non-terminal) tasks by exact URL, and on `take` calls `add_one()` — the same path `download.add` itself uses — so a captured download is a real, admitted, persisted task, not a special case. The 750 ms deadline (CLAUDE.md §4 / AGENT-DAEMON.md build step 6) is checked cooperatively between every step via a new `rpc::CaptureDataSource` seam (real impl wraps `store::*`; a test fake can jump its own clock forward to simulate "the store was slow just now" with zero real sleep) — catches the realistic failure mode (several slow steps adding up) though it can't preempt one pathologically stuck single call. Verified against real `veloxd` + `tools/testserver`: a monitored-type offer answers in ~5ms and actually creates + downloads the task; an unmonitored type, an excluded host, a rule-vetoed host, and a second offer for a still-active URL all answer `ignore` with the right `reason`; a bad category save dir surfaces its real `-32011` rather than being swallowed. New `capture_offer_test` covers all of the above plus the deadline itself (two cases, one per "slow" checkpoint), asserting real wall-clock time barely moves even though the fake clock jumped 2 simulated seconds — proof the check reads the injected clock, not a disguised sleep. | `rpc/capture_data_source.hpp`, `rpc/dispatcher.{hpp,cpp}`, `store/rules.{hpp,cpp}`, `store/categories.{hpp,cpp}`, `store/tasks.{hpp,cpp}` | — | done |
|
||||
| ~~D8~~ | **Closed alongside D7** — `capture.getRules` returns the same settings-backed `enabled`/`monitoredExtensions`/`monitoredMimeTypes`/`minSizeBytes`/`excludedHosts`/`bypassModifier` capture.offer itself reads, so the two can never drift. `rulesVersion` is a constant `1` — there is no persisted revision counter yet (nothing writes `rules.*` outside this process's own lifetime to need one across a restart), and the extension already re-fetches on `event.settings.changed` regardless of what this number does; noted in case a real counter becomes worth adding later. | `rpc/dispatcher.cpp` | `rulesVersion` is a placeholder constant | — |
|
||||
| D1 | Pairing prompt is `EnvAutoApprover` (needs `VELOX_PAIR_AUTO=1`) | `rpc/pairing.hpp`, `main.cpp` | A GUI dialog / `org.freedesktop.Notifications` approver is integration work | Build step 7 (systemd + notifications) |
|
||||
| D2 | `download.probe` → `-32603` | `rpc/dispatcher.cpp` | `download.add` is wired (`fs/safepath` + store, real `-32011`); `download.probe` needs the engine's probe path for `-32013` | probe with the engine link (CORE stage 3 is landed; wire `Engine::probe`) |
|
||||
| D3 | Stub handlers for everything except `session.*`, `download.add/list/get` | `rpc/dispatcher.cpp` | No store behind them yet (categories/queues/rules/settings/limiter/schedule) | Per method, as the store query modules land behind them |
|
||||
| — | **D1, checked this pass, not attempted:** `libdbus-1-dev` (or `libsystemd-dev` for `sd-bus`) has no headers installed in this build environment — only the runtime `.so`s (`dpkg -l`/`apt-cache policy` confirm `libdbus-1-3` present, `libdbus-1-dev` not, "Candidate" available but not installed). A real notification-backed approver needs one of those linked into `veloxd`, which is a new build dependency for `daemon/CMakeLists.txt` (`find_package`/`pkg_check_modules`) and — since packaging manifests need to know about it too — arguably a decision to surface rather than something to reach for silently mid-session. `PairingApprover::approve()` is also still synchronous by shape (its own doc comment already says so: "the real notification-backed approver will run async and is not this shape") — swapping it for the async pattern this session built for `download.probe` (`rpc::TaskActionPort` + the server-layer deferred-reply special-case) is the right shape once there's a real implementation to justify the churn; reshaping the interface with nothing behind it yet would just be churn. Left `EnvAutoApprover` in place rather than build a fragile hand-rolled D-Bus wire client to avoid the missing headers — a broken pairing approver is worse than an honest stub. | `rpc/pairing.hpp` | missing dev headers + an undiscussed new dependency | once `libdbus-1-dev`/`libsystemd-dev` is available and the dependency is approved |
|
||||
| ~~D2~~ | **Closed** — `download.probe` is real on both transports. It's genuinely async (the engine's probe pool, up to the schema's 30s `x-deadlineMs`) and so cannot fit `VeloxDispatcher::on_download_probe`'s synchronous `HandlerResult<T>` return — `uds_server.cpp`/`ws_server.cpp` special-case `"download.probe"` before the generic `dispatch()`, exactly the way they already special-case `session.hello`/`session.subscribe`, and queue the reply whenever the callback fires. `rpc::TaskActionPort::probe_now` (kept in proto/std terms, no `vdm::net::*`, so `veloxd_rpc` never needs `core/include`'s vdm headers) is what both transports call; `sched::Scheduler::probe_now` is the implementation — builds a `vdm::net::ProbeRequest`, runs it on the engine's probe pool, maps a failure to `-32013 ProbeFailed` (with `data.httpStatus` when there was one), and fills `suggestedCategoryId`/`suggestedSaveDir` with a plain extension match against the categories table (not the real rules engine — that's still D3). Verified live: a real probe answers in ~5ms; a bad host maps to `-32013`; a connection issuing a 10s `slow-loris` probe does not block a second connection's `download.list` (answered in ~1ms) — confirms the async design actually keeps the loop free, not just compiles. | `rpc/task_action_port.hpp`, `rpc/{uds_server,ws_server}.{hpp,cpp}`, `sched/scheduler.{cpp,hpp}` | — | done |
|
||||
| D3 | Stub handlers for the rest: `grabber.*`, `media.*` | `rpc/dispatcher.cpp` | HLS/DASH grabber and media-variant support don't exist anywhere in this build yet — a bigger feature than a store-wiring pass | M4 territory, per AGENT-DAEMON.md |
|
||||
| ~~D3d~~ | **Closed — `rules.list`/`rules.upsert`.** New `store/rules.{hpp,cpp}`: `list()` in priority order, `apply(upsert, remove)` in one transaction (an empty `ruleId` generates one; a reprioritisation and a removal land atomically, per the schema's own "never leaves the table in a half-valid state"). Found and fixed along the way: migration `0001`'s `rules` table had no column for `Rule.name` at all — every `rules.list`/`.upsert` call failed outright ("no such column: name") the first time either ran against a real `Db`, unit tests included, since `:memory:` migrates through the same path. Migration `0004` adds it. | `store/rules.{hpp,cpp}`, `rpc/dispatcher.cpp`, `store/migrations/0004_*.sql` | — | done |
|
||||
| ~~D3e~~ | **Closed — `queue.reorder`.** New `store::Queues::reorder`: `taskIds` must be an exact permutation of the queue's current membership (compared as sorted sets) or nothing is written and `-32602` names the queue; a valid permutation rewrites every member's `queue_position` in one transaction. | `store/queues.{hpp,cpp}`, `rpc/dispatcher.cpp` | — | done |
|
||||
| ~~D3f~~ | **Closed — `schedule.get`/`schedule.set`.** Thin wrapper over the `queues.schedule` column that already existed (`Queue.schema.json`'s own field, read since D3b but never independently settable). `nextRunAt` is deliberately left unset: computing it correctly needs the same local-time, DST-aware window logic `sched/schedule_window.hpp`'s `window_open()` only has half of (is-it-open-right-now, not next-transition) — real work, called out rather than approximated. The field is optional; `nullopt` is a legal answer. | `store/queues.{hpp,cpp}`, `rpc/dispatcher.cpp` | `nextRunAt` unset (documented, not silently wrong) | `nextRunAt` is its own pass |
|
||||
| ~~D3g~~ | **Closed — `limiter.get`/`limiter.set`.** Backed by the same `downloads.speedLimitEnabled`/`downloads.speedLimitBps` settings keys D9 already wired (one bag of truth, not two) — the *new* part is actually reaching the engine: `EnginePort`/`TaskActionPort` gain `set_global_speed_limit(bps)` (`0` = unlimited, `vdm::rate::TokenBucket`'s own convention), wired to `vdm::Engine::rate_limiter().set_global_limit()`. Pushed live on every `limiter.set` *and* on `Scheduler::reload_config()` (so a limit from a previous run isn't silently unlimited again after a restart — nothing else re-derives it from settings the way `connection.*` already does). `applyToRunning` is accepted but has no lever to pull differently: a single shared global bucket has no "next task only" variant, so this always behaves as if it were `true` — documented in `EnginePort::set_global_speed_limit`'s own comment. Also found, not chased further: the schema's "`globalBps: 0` with `enabled: true` means 'stop everything'" is the *opposite* of what `TokenBucket` does with `rate_bps == 0` (unlimited) — a real discrepancy, but the schema itself says the GUI "must not offer" that combination, so nothing sends it in practice. | `sched/engine_port*.hpp`, `rpc/task_action_port.hpp`, `sched/scheduler.{cpp,hpp}`, `rpc/dispatcher.cpp` | the `globalBps:0` semantic clash noted above; `applyToRunning` has no real effect | flagged for whoever owns the schema/CORE conversation next |
|
||||
| ~~D3h~~ | **Closed — `download.update`.** "Moving `saveDir` or `filename` moves the file on disk in the same operation" (the schema's own words): resolved and root-checked exactly like `download.add`'s destination, then the `.veloxpart`/`.veloxpart.meta` pair (or the finished file, if the task is `complete`) is moved with `std::filesystem::rename`, falling back to copy+remove across filesystems (`EXDEV`) — only if the resolved location actually differs from where the task already is. `categoryId`/`queueId`(appended to the end of the new queue's run order)/`description`/`segments`(1-32)/`bufferBytes`(64 KiB-16 MiB)/`checksum` all apply through a new `store::Tasks::apply_update`. New `store::Tasks::UpdatePatch`/`apply_update` and `Tasks::set_url` (download.refreshUrl's own need, split out since neither `apply_update` nor `set_probe_result` owns the base `url` column). | `store/tasks.{hpp,cpp}`, `rpc/dispatcher.cpp` | see the shared note below (null-clearing) | done |
|
||||
| ~~D3i~~ | **Closed — `download.refreshUrl`.** Same async reasoning and the same server-layer special-case (`uds_server.cpp`/`ws_server.cpp` intercept before generic `dispatch()`, exactly like `download.probe`) — a real network round trip, same 30s `x-deadlineMs`. Re-probes the new URL, flags `contentChanged` only when size or validator are *both* known and actually differ (an unknown value on either side is never itself a mismatch — "it says so rather than silently restarting" needs a real disagreement, not an absent comparison), persists the new URL and probe result, and — if the task holds a live engine handle — swaps the URL in place via a newly-widened `EnginePort::refresh_url` (now takes headers too, matching `DownloadHandle::refresh_url`'s real signature; the seam had silently dropped them). Verified against real `veloxd` + `tools/testserver`, live-handle swap covered by a `sched_scheduler_test` case (asserts the engine got `refresh_url()`, not a fresh `start()`). | `sched/engine_port*.hpp`, `rpc/task_action_port.hpp`, `sched/scheduler.{cpp,hpp}`, `rpc/{uds_server,ws_server}.{hpp,cpp}` | — | done |
|
||||
| — | **Shared note across D3h/download.update and D9/settings.set:** the generated parser collapses "field absent" and "field explicitly `null`" to the same `std::optional::nullopt` for every `optional<T>` patch field (`DownloadUpdateParamsPatch`, `Settings`) — there is no second bit on the wire path that survives into `VeloxDispatcher`. Both schemas document "an explicit null clears the field," but neither handler can act on that distinction because the information is already gone by the time either sees the parsed struct. Not something to route around locally (would mean hand-parsing raw JSON past the generated `parse<T>` for a handful of fields) — this is a generator-level gap, PROTO's to close (e.g. `std::optional<std::optional<T>>`, or a parallel "which fields were present" bitset). Until then: `categoryId`/`queueId`/`description`/`checksum` on `download.update`, and every nullable `Settings` key, can be *set* through these RPCs but never explicitly cleared back to null. | `contracts/` (generator), affects `rpc/dispatcher.cpp` | the wire distinction the schema documents doesn't survive to the handler | PROTO's generator |
|
||||
| ~~D9~~ | **Closed — `settings.get`/`settings.set`.** The field <-> `SettingKey` <-> JSON-type mapping is four pointer-to-member tables in `dispatcher.cpp` (one per C++ field type: bool, ranged int, plain string, string array) plus five enum-typed keys handled individually (`parse_XXX` already validates those); every one of the 43 `SettingKey`s now has a real default (`store::Settings::kDefaults` grew from 14 entries to 43 — `capture.monitoredExtensions`'s default is the union of every builtin category's extensions, so the two never drift apart). `settings.get` honors `keys: null` = everything. `settings.set` validates every field *before* writing any of them (numeric min/max — the schema itself carries none of this, so it's hand-checked against each key's documented range; `-32602` names the offending key, its value, and its bounds) and validates `saveTo.*` paths against `fs::resolve_target`/`canonicalize_root` (`-32011`) — `saveTo.allowedRoots` entries are checked as roots in their own right, `saveTo.defaultDir`/`.tempDir` are checked as paths resolving *inside* the (possibly, in the same call, just-updated) root list. Reports exactly the keys whose *effective* value actually changed (a `set` to the value already in effect reports `changed: []`, not the key), publishes `event.settings.changed` with that same list, and calls `TaskActionPort::apply_settings_reload()` (-> `Scheduler::reload_config()`) when any `connection.*` key took effect, live rather than waiting for a restart. Verified against real `veloxd`: all 43 keys round-trip with sane defaults, a `keys` subset filters correctly, an out-of-range value is rejected with nothing else in the same call landing, `saveTo.defaultDir` outside every allowed root is `-32011`, setting `allowedRoots` and `defaultDir` together cross-validates against the *new* roots, and `event.settings.changed` fires over a live subscription. New `dispatcher_settings_test` covers the same ground without a socket. | `rpc/dispatcher.cpp`, `store/settings.{hpp,cpp}`, `rpc/task_action_port.hpp`, `sched/scheduler.hpp` | — | done |
|
||||
| ~~D3a~~ | **Closed** — `category.upsert`/`category.remove`: `store/categories.hpp` gains `get`/`upsert`/`remove`. `upsert` generates an id when absent (create) and always ignores the payload's `builtin` (preserved from the existing row on replace, false on create — a client can never mint or revoke it); the `saveDir` goes through the same `fs::resolve_target` canonicalize-and-root-check as `download.add` (`-32011` on failure). `remove` refuses a builtin at both layers (dispatcher pre-checks for the `-32602` error text; the store's own `DELETE ... AND builtin = 0` is defense in depth) and reassigns member tasks to `reassignTo` (default `"general"`) inside one transaction before deleting the row. Note: the `categories` table (0001) has no columns for `Category.mimeTypes`/`.sortOrder` — accepted on `upsert` but not persisted. | `store/categories.{hpp,cpp}`, `rpc/dispatcher.cpp` | — | done, `mimeTypes`/`sortOrder` gap noted |
|
||||
| ~~D3b~~ | **Closed** — `queue.upsert`: `store/queues.hpp` gains `get`/`upsert` (`set_state` already existed from D4b). Same create-generates-id pattern as categories; `taskIds` in the payload is ignored (schema's own note) and a create always starts `'stopped'` while a replace keeps the queue's current run state — `queue.upsert` edits config, not run state (that's `queue.start`/`stop`). Also fixed: `on_complete` was a real column since 0001 but `Queues::list`/`get` never projected it onto `Queue.onComplete` — now they do. | `store/queues.{hpp,cpp}`, `rpc/dispatcher.cpp` | — | done |
|
||||
| ~~D3c~~ | **Closed** — `download.remove`: cancels with `discard_partial=true` through `TaskActionPort` (always drops any `.veloxpart`/`.veloxpart.meta` — the row is gone either way, unlike `download.cancel`, which keeps them), deletes the finished file only when `deleteFile` is true and the task was `complete` (best-effort — a missing file doesn't fail the call), deletes the row (segments cascade via the FK), and publishes `event.task.removed` (closing the last open note under D5). `download.addBatch`: `on_download_add`'s body is now a shared `add_one()`, called once per item after merging each item's unset fields against `params.defaults`. `download.provideAuth`: forwards to `EnginePort::provide_auth` through a new `TaskActionPort::provide_auth`; `remember`/persisting to the Secret Service is accepted but not acted on — nothing in this build talks to libsecret yet (verified: no such integration exists anywhere in the tree). Verified against real `veloxd` + `tools/testserver`: category create/replace/remove-with-reassignment, queue create/replace-keeps-state, a batch add with shared `defaults.saveDir`, and remove-with-deleteFile actually deleting the file and the task then 404ing `download.get` with `-32010`. | `rpc/dispatcher.{hpp,cpp}`, `rpc/task_action_port.hpp`, `sched/scheduler.{cpp,hpp}` | `download.provideAuth`'s `remember` (needs the Secret Service, unbuilt) | done, `remember` persistence gap noted |
|
||||
| ~~D4a~~ | **Closed** — `sched/engine_port_core.hpp` wraps `vdm::Engine` + `segment_budget()`; `main.cpp` constructs `Engine` + `Scheduler`, calls `reconcile_after_restart` / `reload_config` / `tick` at startup | — | — | done (`lane/core` stage 8 merged) |
|
||||
| D4b | timer + nudges: a 1 s `timerfd` re-runs `Scheduler::tick()` and `download.add` nudges via `on_mutation`. `download.pause`/`resume`/`start`/`cancel` and the queue.* handlers still don't touch the scheduler | `rpc/dispatcher.cpp` | those handlers are still stubs (D3) | as each handler is implemented behind the store, it calls `on_mutation` / drives the scheduler |
|
||||
| D5 | `event.*` fan-out not implemented; `session.subscribe` accepts and echoes but nothing is emitted | `rpc/uds_server.cpp`, `rpc/ws_server.cpp` | No task state to broadcast until the engine is wired. `Scheduler::on_engine_state` is the hook it will fire from | With D4a — the same engine-state callback feeds both the store and `event.task.state` |
|
||||
| ~~D4b~~ | **Closed** — `download.pause`/`resume`/`start`/`cancel` and `queue.start`/`stop` all drive the scheduler now, and apply *immediately* (not deferred to the next tick — pausing/resuming/cancelling a live transfer can't wait up to 1s, and per ADR 0013 §3 the governor never touches a user-owned pause on its own). New `rpc::TaskActionPort` interface (owned by `rpc/`, implemented by `sched::Scheduler`) is the seam dispatcher.hpp depends on instead of `sched/scheduler.hpp` directly — avoids a real `veloxd_rpc` <-> `veloxd_sched` circular library dependency (`veloxd_sched` already links `veloxd_rpc` for `EventHub`). `Scheduler::user_pause/resume/start/cancel` + `pause_queue` engine-call-then-eager-transition, matching `tick()`'s existing `to_pause` pattern. Fixed a real bug hit while building this: `transition()` always overwrote `pause_reason` to NULL when the engine's own delayed pause-ack callback arrived with no explicit reason, clobbering whatever the actual initiator (user or governor) had just written — now it preserves the stored reason when none is supplied. Verified against real `veloxd` + `tools/testserver`: pausing a live single-segment throttled transfer freezes `downloadedBytes`, resume continues it from that point, cancel stops it; `queue.stop(pauseRunning:true)` pauses the queue's running task immediately. NOTE: `download.start`'s contract "a task in 'queued' jumps its queue" (priority bump) is not implemented — admission is still plain FIFO by `created_at`. | `sched/scheduler.{cpp,hpp}`, `rpc/task_action_port.hpp`, `rpc/dispatcher.{hpp,cpp}`, `store/queues.{cpp,hpp}` | — | done, except the queue-jump priority bump noted above |
|
||||
| ~~D5~~ | **Mostly closed** — `rpc/event_hub` fans out per-subscription; `session.subscribe` on both transports registers/updates/tears down a real subscription; `Scheduler::transition()` publishes `event.task.state` (with `previousState`) on every state change, scheduler-driven or engine-reported; `dispatcher::on_download_add` publishes `event.task.added`; a 250 ms timer batches `Scheduler::progress_snapshot()` into one `event.task.progress` array per AGENT-DAEMON.md item 5 / the schema's `x-maxRateHz: 4`. Verified live end to end. | — | `event.task.removed` has no source yet (`download.remove` is D3); `event.speed.global`, `event.notify`, `event.auth.required`, `event.settings.changed`, `event.grabber.progress` are unpublished — each lands with its owning handler | as each owning D3 handler lands |
|
||||
| ~~D6~~ | **Closed** — engine numbers now reach the store: `Scheduler::tick()` probes (`EnginePort::probe`) before every `start()`, persisting `sizeBytes`/`resumable`/validators via `Tasks::set_probe_result` before a byte moves; `Scheduler::persist_progress()` (called from `progress_snapshot()` *and* once more from `on_engine_state` right before `release()`/unmap on every terminal transition) writes `downloadedBytes`/`speedBps`/`segments`/`segmentDetail` from the engine's `Progress`, so a task that finishes between two 250 ms ticks (the common case for anything small or fast) still leaves real numbers instead of the pre-persistence defaults. `TaskSummary.segments` is sourced from `segments.size()` when the task has any (matching what actually lands in `segmentDetail`, per the schema's "exactly `segments` entries"), falling back to the engine's `effective_segments` (budget slots *held*, not necessarily physical range count — see `core/include/vdm/task/download.hpp`'s `Progress` comment) only pre-segmentation. `Tasks::set_final_bytes` tops up `on_finished`'s byte count as a last-resort backstop. Migration `0002` adds `speed_bps` to both `tasks` and `segments`, and fixes `segments.state`'s CHECK to include `'pending'` (0001 omitted it, so a pre-connect snapshot could never be written). Verified against real `veloxd` + `tools/testserver` (not just unit tests): `download.list`/`download.get` correct immediately after completion and after a daemon restart. | `sched/scheduler.{cpp,hpp}`, `store/{tasks,segments}.{cpp,hpp}`, `store/migrations/0002_*.sql` | — | done |
|
||||
| — | ~~Observed, not fixed (CORE, not this lane)~~ — **routed to CORE by the user.** `vdm::task::Progress.speed_bps` reads back as `0` for the whole lifetime of a live, real (non-fake) throttled download, despite `downloadedBytes` visibly advancing between polls — `core/src/task/download_task.cpp`'s per-worker EWMA never seems to produce a nonzero aggregate in this build. DAEMON passes `EnginePort::progress()`'s `speed_bps` straight through (`Scheduler::persist_progress`); nothing in this lane drops it. Still reproduces in the D4b live checks above (0 throughout a paused/resumed/cancelled transfer whose `downloadedBytes` visibly moved) — not re-filed, since it's already CORE's. |
|
||||
|
||||
+59
-34
@@ -17,10 +17,14 @@
|
||||
|
||||
#include <sys/timerfd.h>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/event_loop.hpp"
|
||||
#include "rpc/pairing.hpp"
|
||||
#include "rpc/runtime_dir.hpp"
|
||||
#include "rpc/single_instance.hpp"
|
||||
#include "rpc/uds_server.hpp"
|
||||
#include "rpc/ws_server.hpp"
|
||||
#include "sched/engine_port_core.hpp"
|
||||
@@ -28,6 +32,7 @@
|
||||
#include "sched/scheduler.hpp"
|
||||
#include "store/migrations.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "util/time.hpp"
|
||||
#include "vdm/engine.hpp"
|
||||
#include "version.hpp"
|
||||
|
||||
@@ -39,48 +44,25 @@ void on_signal(int) {
|
||||
if (g_loop != nullptr) g_loop->stop(); // stop() is async-signal-safe (writes an eventfd)
|
||||
}
|
||||
|
||||
// Single-instance guard: bind an abstract-namespace Unix socket whose name is unique to
|
||||
// this user. A second daemon gets EADDRINUSE and exits. The kernel reclaims an
|
||||
// abstract-namespace address when the holding process dies, so a crash never wedges it
|
||||
// (docs/01 §2). Returns the held fd (kept open for the process lifetime) or -1.
|
||||
int acquire_single_instance_lock() {
|
||||
const int fd = ::socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0);
|
||||
if (fd < 0) return -1;
|
||||
|
||||
const std::string name = std::string("velox-daemon-") + std::to_string(::geteuid());
|
||||
sockaddr_un addr{};
|
||||
addr.sun_family = AF_UNIX;
|
||||
// Leading NUL selects the abstract namespace; the name follows, not NUL-terminated.
|
||||
addr.sun_path[0] = '\0';
|
||||
std::memcpy(addr.sun_path + 1, name.c_str(), name.size());
|
||||
const socklen_t len =
|
||||
static_cast<socklen_t>(offsetof(sockaddr_un, sun_path) + 1 + name.size());
|
||||
|
||||
if (::bind(fd, reinterpret_cast<sockaddr*>(&addr), len) != 0) {
|
||||
::close(fd);
|
||||
return -1;
|
||||
}
|
||||
return fd;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
std::cout << "veloxd " << velox::daemon::kDaemonVersion << " (protocol "
|
||||
<< velox::proto::kProtocolVersion << ")\n";
|
||||
|
||||
const int lock_fd = acquire_single_instance_lock();
|
||||
if (lock_fd < 0) {
|
||||
std::cerr << "veloxd: another instance is already running for this user\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
velox::daemon::rpc::RuntimeDir rt;
|
||||
if (const auto ec = velox::daemon::rpc::resolve_runtime_dir(rt)) {
|
||||
std::cerr << "veloxd: cannot prepare runtime directory: " << ec.message() << "\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
const int lock_fd = velox::daemon::rpc::acquire_single_instance_lock(rt.path);
|
||||
if (lock_fd < 0) {
|
||||
std::cerr << "veloxd: another instance is already running for this runtime "
|
||||
"directory (" << rt.path << ")\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
velox::daemon::rpc::EventLoop loop;
|
||||
g_loop = &loop;
|
||||
|
||||
@@ -108,10 +90,11 @@ int main() {
|
||||
}
|
||||
|
||||
// --- engine + scheduler ---------------------------------------------------------
|
||||
velox::daemon::rpc::EventHub hub;
|
||||
vdm::Engine engine;
|
||||
velox::daemon::sched::EnginePortCore engine_port(engine);
|
||||
velox::daemon::sched::Scheduler scheduler(
|
||||
*db, engine_port, velox::daemon::sched::Governor{},
|
||||
*db, engine_port, velox::daemon::sched::Governor{}, &hub,
|
||||
{/*local_now*/ {},
|
||||
/*post_to_loop*/ [&loop](std::function<void()> fn) { loop.post(std::move(fn)); }});
|
||||
|
||||
@@ -120,7 +103,7 @@ int main() {
|
||||
(void)scheduler.reload_config();
|
||||
(void)scheduler.tick(); // admit anything already queued in the DB
|
||||
|
||||
velox::daemon::rpc::VeloxDispatcher dispatcher(*db);
|
||||
velox::daemon::rpc::VeloxDispatcher dispatcher(*db, hub, &scheduler);
|
||||
dispatcher.set_on_mutation([&loop, &scheduler] {
|
||||
loop.post([&scheduler] { (void)scheduler.tick(); });
|
||||
});
|
||||
@@ -140,7 +123,45 @@ int main() {
|
||||
});
|
||||
}
|
||||
|
||||
velox::daemon::rpc::UdsServer uds(loop, dispatcher, rt.socket_path());
|
||||
// event.task.progress: one array message at <=4 Hz (schema x-maxRateHz), never one
|
||||
// notification per task (AGENT-DAEMON.md item 5). 250 ms keeps every active task's
|
||||
// segment bar under 4 Hz without depending on how many tasks are running.
|
||||
const int progress_fd = ::timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC);
|
||||
if (progress_fd >= 0) {
|
||||
itimerspec spec{};
|
||||
spec.it_value.tv_nsec = 250'000'000;
|
||||
spec.it_interval.tv_nsec = 250'000'000;
|
||||
::timerfd_settime(progress_fd, 0, &spec, nullptr);
|
||||
loop.add_fd(progress_fd, velox::daemon::rpc::kRead, [&](int fd, unsigned) {
|
||||
std::uint64_t ticks = 0;
|
||||
[[maybe_unused]] ssize_t n = ::read(fd, &ticks, sizeof(ticks));
|
||||
const auto rows = scheduler.progress_snapshot();
|
||||
if (rows.empty()) return;
|
||||
|
||||
nlohmann::json tasks_json = nlohmann::json::array();
|
||||
for (const auto& r : rows) {
|
||||
nlohmann::json t{{"taskId", r.task_id},
|
||||
{"downloadedBytes", r.downloaded_bytes},
|
||||
{"speedBps", r.speed_bps}};
|
||||
t["etaSeconds"] = r.eta_seconds ? nlohmann::json(*r.eta_seconds) : nlohmann::json(nullptr);
|
||||
if (!r.segments.empty()) {
|
||||
nlohmann::json segs = nlohmann::json::array();
|
||||
for (const auto& s : r.segments)
|
||||
segs.push_back({{"index", s.index},
|
||||
{"downloadedBytes", s.downloaded_bytes},
|
||||
{"speedBps", s.speed_bps}});
|
||||
t["segments"] = std::move(segs);
|
||||
}
|
||||
tasks_json.push_back(std::move(t));
|
||||
}
|
||||
const nlohmann::json params{{"tasks", std::move(tasks_json)},
|
||||
{"at", velox::daemon::now_iso()}};
|
||||
hub.publish(velox::proto::Event::TaskProgress,
|
||||
velox::proto::make_notification(velox::proto::Event::TaskProgress, params));
|
||||
});
|
||||
}
|
||||
|
||||
velox::daemon::rpc::UdsServer uds(loop, dispatcher, hub, rt.socket_path(), &scheduler);
|
||||
if (const auto ec = uds.start()) {
|
||||
std::cerr << "veloxd: cannot listen on " << rt.socket_path() << ": " << ec.message()
|
||||
<< "\n";
|
||||
@@ -154,7 +175,7 @@ int main() {
|
||||
// TODO(build step 7): replace EnvAutoApprover with a GUI-dialog / desktop-notification
|
||||
// approver. Until then pairing needs VELOX_PAIR_AUTO=1.
|
||||
velox::daemon::rpc::EnvAutoApprover approver;
|
||||
velox::daemon::rpc::WsServer ws(loop, dispatcher, *db, approver, rt);
|
||||
velox::daemon::rpc::WsServer ws(loop, dispatcher, *db, approver, hub, rt, &scheduler);
|
||||
if (const auto ec = ws.start()) {
|
||||
std::cerr << "veloxd: WebSocket transport unavailable (" << ec.message()
|
||||
<< "); the extension fallback will not work this run\n";
|
||||
@@ -169,6 +190,10 @@ int main() {
|
||||
loop.del_fd(tick_fd);
|
||||
::close(tick_fd);
|
||||
}
|
||||
if (progress_fd >= 0) {
|
||||
loop.del_fd(progress_fd);
|
||||
::close(progress_fd);
|
||||
}
|
||||
g_loop = nullptr;
|
||||
::close(lock_fd);
|
||||
return 0;
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
#pragma once
|
||||
|
||||
// The seam capture.offer's decision logic reads through, instead of touching store::* (or
|
||||
// the wall clock) directly — the same reasoning as rpc::TaskActionPort: it lets a test
|
||||
// substitute a fake that reports "the store took a long time just now" by advancing a
|
||||
// shared fake clock, and assert that capture.offer's deadline check actually bails to
|
||||
// `ignore` instead of pressing on, without a real sleep anywhere (deterministic, instant).
|
||||
//
|
||||
// vdm::rules::Rule (not velox::proto::Rule) on purpose: this is what
|
||||
// vdm::rules::match_rules consumes directly, so the real implementation is the only place
|
||||
// that ever converts the stored proto::Rule/JSON shape into CORE's plain vocabulary.
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "vdm/rules/match.hpp"
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
class CaptureDataSource {
|
||||
public:
|
||||
virtual ~CaptureDataSource() = default;
|
||||
|
||||
// Read at every checkpoint in the offer's decision pipeline; a fake can advance this
|
||||
// however it likes (including "jump forward 2 real seconds, instantly") to simulate a
|
||||
// slow step without ever calling sleep.
|
||||
virtual std::chrono::steady_clock::time_point now() = 0;
|
||||
|
||||
virtual bool capture_enabled() = 0;
|
||||
virtual std::vector<std::string> monitored_extensions() = 0;
|
||||
virtual std::vector<std::string> monitored_mime_types() = 0;
|
||||
virtual std::int64_t min_size_bytes() = 0;
|
||||
virtual std::vector<std::string> excluded_hosts() = 0;
|
||||
|
||||
// Enabled rules, in priority order — ready for vdm::rules::match_rules as-is.
|
||||
virtual std::vector<vdm::rules::Rule> enabled_rules() = 0;
|
||||
|
||||
// "resolve the category folder": a plain extension guess (store::Categories'
|
||||
// guess_by_extension) when no rule named a category explicitly.
|
||||
virtual std::string guess_category_id(const std::string& filename) = 0;
|
||||
virtual std::string category_save_dir(const std::string& category_id) = 0;
|
||||
virtual std::string default_save_dir() = 0;
|
||||
|
||||
// True if an active (non-terminal) task already targets this exact URL.
|
||||
virtual bool has_active_duplicate(const std::string& url) = 0;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
+1063
-56
File diff suppressed because it is too large
Load Diff
@@ -14,6 +14,9 @@
|
||||
|
||||
#include <functional>
|
||||
|
||||
#include "rpc/capture_data_source.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/task_action_port.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
@@ -21,12 +24,26 @@ namespace velox::daemon::rpc {
|
||||
|
||||
class VeloxDispatcher final : public velox::proto::Dispatcher {
|
||||
public:
|
||||
explicit VeloxDispatcher(velox::daemon::store::Db& db) : db_(db) {}
|
||||
// `actions` drives download.pause/resume/start/cancel and queue.start/stop
|
||||
// immediately (D4b) — those cannot wait for the next tick(), unlike download.add's
|
||||
// on_mutation nudge. Optional so existing tests that only exercise download.add/list/
|
||||
// get keep building with no scheduler at hand; a null actions_ makes those methods
|
||||
// answer "not implemented" instead of crashing. See rpc/task_action_port.hpp for why
|
||||
// this is an interface owned by rpc/ rather than a direct sched::Scheduler* (avoids a
|
||||
// veloxd_rpc <-> veloxd_sched circular library dependency).
|
||||
VeloxDispatcher(velox::daemon::store::Db& db, EventHub& hub, TaskActionPort* actions = nullptr)
|
||||
: db_(db), hub_(hub), actions_(actions) {}
|
||||
|
||||
// Called after a handler mutates task state (download.add for now). main.cpp wires it
|
||||
// to nudge the scheduler; unset in tests.
|
||||
void set_on_mutation(std::function<void()> fn) { on_mutation_ = std::move(fn); }
|
||||
|
||||
// Test-only seam: capture.offer normally builds its own real CaptureDataSource
|
||||
// (wrapping db_) per call. A test that needs to simulate "the store is slow right
|
||||
// now" (see rpc/capture_data_source.hpp) supplies one here instead; production code
|
||||
// never calls this.
|
||||
void set_capture_source_for_test(CaptureDataSource* src) { capture_source_for_test_ = src; }
|
||||
|
||||
velox::proto::HandlerResult<velox::proto::CaptureRules>
|
||||
on_capture_getRules(const velox::proto::CaptureGetRulesParams&) override;
|
||||
velox::proto::HandlerResult<velox::proto::CaptureOfferResult>
|
||||
@@ -106,8 +123,17 @@ public:
|
||||
on_settings_set(const velox::proto::SettingsSetParams&) override;
|
||||
|
||||
private:
|
||||
// The whole of download.add's body; on_download_add and on_download_addBatch (each
|
||||
// item merged against DownloadAddBatchParams.defaults first) both call this — exactly
|
||||
// one place turns a DownloadSpec into a stored, admitted task.
|
||||
velox::proto::HandlerResult<velox::proto::DownloadAddResult> add_one(
|
||||
const velox::proto::DownloadSpec& spec);
|
||||
|
||||
velox::daemon::store::Db& db_;
|
||||
EventHub& hub_;
|
||||
TaskActionPort* actions_;
|
||||
std::function<void()> on_mutation_;
|
||||
CaptureDataSource* capture_source_for_test_ = nullptr;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
#include "rpc/event_hub.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
namespace proto = velox::proto;
|
||||
|
||||
EventHub::SubId EventHub::subscribe(Sink sink) {
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
const SubId id = next_++;
|
||||
subs_.emplace(id, Sub{std::move(sink), {}, std::nullopt});
|
||||
return id;
|
||||
}
|
||||
|
||||
void EventHub::set_filter(SubId id, std::vector<proto::Event> events,
|
||||
std::optional<std::vector<std::string>> task_ids) {
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
if (auto it = subs_.find(id); it != subs_.end()) {
|
||||
it->second.events = std::move(events);
|
||||
it->second.task_ids = std::move(task_ids);
|
||||
}
|
||||
}
|
||||
|
||||
void EventHub::unsubscribe(SubId id) {
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
subs_.erase(id);
|
||||
}
|
||||
|
||||
void EventHub::publish(proto::Event kind, const nlohmann::json& notification,
|
||||
std::string_view task_id) {
|
||||
// Copy the sinks to call out to while holding the lock only long enough to build the
|
||||
// list — a sink runs arbitrary connection code (framing + a write syscall) and must
|
||||
// not run with mu_ held.
|
||||
std::vector<Sink> targets;
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
targets.reserve(subs_.size());
|
||||
for (const auto& [id, sub] : subs_) {
|
||||
(void)id;
|
||||
if (std::find(sub.events.begin(), sub.events.end(), kind) == sub.events.end())
|
||||
continue;
|
||||
if (!task_id.empty() && sub.task_ids &&
|
||||
std::find(sub.task_ids->begin(), sub.task_ids->end(), task_id) ==
|
||||
sub.task_ids->end())
|
||||
continue;
|
||||
targets.push_back(sub.sink);
|
||||
}
|
||||
}
|
||||
for (const auto& sink : targets) sink(notification);
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
@@ -0,0 +1,63 @@
|
||||
#pragma once
|
||||
|
||||
// Per-subscription event fan-out, shared by both transports. A connection subscribes once
|
||||
// (session.subscribe) with the event kinds and optional task-id filter it wants; publish()
|
||||
// delivers a pre-built notification to every subscription that asked for that kind and
|
||||
// passes the per-task filter.
|
||||
//
|
||||
// event.task.progress is the one call site that matters for load: it is batched by the
|
||||
// caller (Scheduler::progress_snapshot + one publish) into a single array message at
|
||||
// <=4 Hz, never one publish per task — that batching happens before this class ever sees
|
||||
// it (AGENT-DAEMON.md item 5, event.task.progress.schema.json x-maxRateHz).
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
class EventHub {
|
||||
public:
|
||||
using SubId = std::uint64_t;
|
||||
using Sink = std::function<void(const nlohmann::json&)>;
|
||||
|
||||
// Register a connection with no interest yet; session.subscribe calls set_filter to
|
||||
// actually turn events on. Returns the id to unsubscribe with on disconnect.
|
||||
SubId subscribe(Sink sink);
|
||||
|
||||
// Replaces the subscription's event set and task filter (session.subscribe replaces,
|
||||
// never adds — matches the method's own description).
|
||||
void set_filter(SubId id, std::vector<velox::proto::Event> events,
|
||||
std::optional<std::vector<std::string>> task_ids);
|
||||
|
||||
void unsubscribe(SubId id);
|
||||
|
||||
// `notification` is a complete {jsonrpc, method, params} object
|
||||
// (velox::proto::make_notification). `task_id` is matched against each subscription's
|
||||
// filter when set; empty means "not task-scoped" and reaches every subscriber of
|
||||
// `kind` regardless of their filter.
|
||||
void publish(velox::proto::Event kind, const nlohmann::json& notification,
|
||||
std::string_view task_id = {});
|
||||
|
||||
private:
|
||||
struct Sub {
|
||||
Sink sink;
|
||||
std::vector<velox::proto::Event> events;
|
||||
std::optional<std::vector<std::string>> task_ids;
|
||||
};
|
||||
|
||||
std::mutex mu_;
|
||||
std::unordered_map<SubId, Sub> subs_;
|
||||
SubId next_ = 1;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
@@ -0,0 +1,38 @@
|
||||
#include "rpc/single_instance.hpp"
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstring>
|
||||
|
||||
#include <sys/socket.h>
|
||||
#include <sys/un.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "util/crypto.hpp"
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
int acquire_single_instance_lock(const std::string& runtime_dir) {
|
||||
const int fd = ::socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0);
|
||||
if (fd < 0) return -1;
|
||||
|
||||
// Truncated to 16 hex chars (64 bits): a collision would need two distinct runtime
|
||||
// dirs to hash together, which is not a security boundary here — the socket itself is
|
||||
// still 0600 same-UID-checked; this is only "don't let two daemons stomp each other".
|
||||
const std::string name =
|
||||
"velox-daemon-" + velox::daemon::crypto::sha256_hex(runtime_dir).substr(0, 16);
|
||||
sockaddr_un addr{};
|
||||
addr.sun_family = AF_UNIX;
|
||||
// Leading NUL selects the abstract namespace; the name follows, not NUL-terminated.
|
||||
addr.sun_path[0] = '\0';
|
||||
std::memcpy(addr.sun_path + 1, name.c_str(), name.size());
|
||||
const socklen_t len =
|
||||
static_cast<socklen_t>(offsetof(sockaddr_un, sun_path) + 1 + name.size());
|
||||
|
||||
if (::bind(fd, reinterpret_cast<sockaddr*>(&addr), len) != 0) {
|
||||
::close(fd);
|
||||
return -1;
|
||||
}
|
||||
return fd;
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
@@ -0,0 +1,27 @@
|
||||
#pragma once
|
||||
|
||||
// Single-instance guard: bind an abstract-namespace Unix socket whose name is derived
|
||||
// from the canonical runtime directory (resolve_runtime_dir's result — already the
|
||||
// per-user default, /run/user/<uid>/velox, unless XDG_RUNTIME_DIR says otherwise). A
|
||||
// second daemon pointed at the same runtime dir gets EADDRINUSE and exits; one pointed at
|
||||
// a different (isolated / test) runtime dir gets its own lock and starts fine. The kernel
|
||||
// reclaims an abstract-namespace address when the holding process dies, so a crash never
|
||||
// wedges it (docs/01 §2).
|
||||
//
|
||||
// Naming this "velox-daemon-<euid>" alone (the old scheme) meant exactly one name per
|
||||
// user system-wide, so XDG_RUNTIME_DIR isolation never reached it: a leaked test veloxd
|
||||
// with the same euid held the lock for every isolated instance too, real or test, until
|
||||
// it was killed. Hashing the resolved runtime dir path instead keeps the real per-user
|
||||
// daemon unique (its runtime dir is unique to it) while letting isolated instances that
|
||||
// each point at their own runtime dir coexist.
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
// Returns the held fd (kept open for the process lifetime; closing it releases the lock)
|
||||
// or -1 if another process already holds the lock for this exact `runtime_dir`, or on any
|
||||
// other socket error.
|
||||
int acquire_single_instance_lock(const std::string& runtime_dir);
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
@@ -0,0 +1,96 @@
|
||||
#pragma once
|
||||
|
||||
// The seam between the dispatcher and the scheduler for user-initiated task/queue actions
|
||||
// (download.pause/resume/start/cancel, queue.stop's pauseRunning) — owned by rpc/ so
|
||||
// dispatcher.hpp (part of veloxd_rpc) never has to include sched/scheduler.hpp, which
|
||||
// would make veloxd_rpc depend on veloxd_sched at compile time. veloxd_sched already
|
||||
// depends on veloxd_rpc (for EventHub); the other direction too would be a real circular
|
||||
// library dependency, not just an inconvenience — anything linking veloxd_rpc alone (e.g.
|
||||
// the CLI's tests) would fail to link over symbols it never calls.
|
||||
//
|
||||
// sched::Scheduler implements this directly (it already lives in a library that depends on
|
||||
// rpc/, so adding an rpc-defined base costs nothing new); main.cpp hands the dispatcher a
|
||||
// `TaskActionPort*` pointing at the same Scheduler it constructs.
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
class TaskActionPort {
|
||||
public:
|
||||
virtual ~TaskActionPort() = default;
|
||||
|
||||
// Mirrors sched::Scheduler::UserActionResult: whether the task was found at all,
|
||||
// whether it actually changed state (a task already in the target/a terminal state is
|
||||
// reported found=true, changed=false — BulkTaskResult's own "not an error" contract),
|
||||
// and its resulting/current state spelling either way.
|
||||
struct Result {
|
||||
bool found = false;
|
||||
bool changed = false;
|
||||
std::string state;
|
||||
};
|
||||
|
||||
virtual Result user_pause(const std::string& wire_id) = 0;
|
||||
virtual Result user_resume(const std::string& wire_id) = 0;
|
||||
virtual Result user_start(const std::string& wire_id) = 0;
|
||||
virtual Result user_cancel(const std::string& wire_id, bool discard_partial) = 0;
|
||||
|
||||
// queue.stop(pauseRunning=true): pause every currently-running task in `queue_id` now.
|
||||
// Returns the wire ids actually paused.
|
||||
virtual std::vector<std::string> pause_queue(const std::string& queue_id) = 0;
|
||||
|
||||
// download.provideAuth: answers a task auto-paused on a 401/407. false if the task
|
||||
// isn't currently holding a live engine handle (nothing waiting on credentials).
|
||||
// `remember` is accepted but not yet acted on — persisting to the Secret Service isn't
|
||||
// wired anywhere in this build yet (CLAUDE.md §4: never SQLite, never logs); this
|
||||
// always does the "this retry only" half. Noted in deferrals.md.
|
||||
virtual bool provide_auth(const std::string& wire_id, const std::string& username,
|
||||
const std::string& password, bool remember) = 0;
|
||||
|
||||
// download.probe (D2), the File Info dialog's own network round trip — no task row
|
||||
// involved. Genuinely async (the engine's probe pool; up to the schema's 30s
|
||||
// x-deadlineMs) and so cannot fit VeloxDispatcher's synchronous on_download_probe:
|
||||
// the RPC server layer (uds_server.cpp / ws_server.cpp) special-cases "download.probe"
|
||||
// before the generic dispatch(), the same way it already special-cases session.hello,
|
||||
// calls this, and queues the reply whenever `done` fires — on an engine thread, so the
|
||||
// implementation must marshal back to the loop before calling it, the same as every
|
||||
// other EnginePort callback. Kept in std::string/proto terms (not vdm::net::*) so this
|
||||
// header — included by dispatcher.hpp, part of veloxd_rpc — never needs core/include's
|
||||
// vdm headers; the vdm::net::ProbeRequest/ProbeResult conversion lives in sched/, which
|
||||
// already depends on vdm.
|
||||
virtual void probe_now(
|
||||
const velox::proto::DownloadProbeParams& params,
|
||||
std::function<void(velox::proto::HandlerResult<velox::proto::DownloadProbeResult>)>
|
||||
done) = 0;
|
||||
|
||||
// download.refreshUrl ("IDM's 'Refresh Download Address'"): same async reasoning and
|
||||
// the same server-layer special-case as probe_now — a real network round trip, up to
|
||||
// the schema's own 30s x-deadlineMs. Re-probes the new URL, compares size/validator
|
||||
// against what the task already has on record (contentChanged), persists the new URL
|
||||
// and probe result, and — if the task holds a live engine handle — swaps its URL
|
||||
// in-flight without losing progress.
|
||||
virtual void refresh_url(
|
||||
const std::string& wire_id, const std::string& url,
|
||||
const std::optional<velox::proto::Headers>& headers,
|
||||
const std::optional<std::vector<velox::proto::Cookie>>& cookies,
|
||||
std::function<void(velox::proto::HandlerResult<velox::proto::DownloadRefreshUrlResult>)>
|
||||
done) = 0;
|
||||
|
||||
// settings.set of a connection.* key: re-read connection.maxConcurrentDownloads /
|
||||
// maxActiveSegments and the per-host cap table, push the new caps to the engine and
|
||||
// the governor (Scheduler::reload_config's own doc comment names this exact trigger).
|
||||
// Named apply_settings_reload rather than reload_config to avoid colliding with
|
||||
// sched::Scheduler's own already-public reload_config() (returns DbResult<void>,
|
||||
// consumed by main.cpp and a unit test — kept as-is rather than reshaped to fit here).
|
||||
virtual void apply_settings_reload() = 0;
|
||||
|
||||
// limiter.set: pushed straight to the engine's shared global token bucket. 0 means
|
||||
// unlimited (the bucket's own convention).
|
||||
virtual void set_global_speed_limit(std::uint64_t bps) = 0;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
@@ -56,8 +56,10 @@ json rpc_error(const json& id, proto::ErrorCode code, std::string_view msg, json
|
||||
|
||||
} // namespace
|
||||
|
||||
UdsServer::UdsServer(EventLoop& loop, proto::Dispatcher& dispatcher, std::string socket_path)
|
||||
: loop_(loop), dispatcher_(dispatcher), path_(std::move(socket_path)) {}
|
||||
UdsServer::UdsServer(EventLoop& loop, proto::Dispatcher& dispatcher, EventHub& hub,
|
||||
std::string socket_path, TaskActionPort* actions)
|
||||
: loop_(loop), dispatcher_(dispatcher), hub_(hub), actions_(actions),
|
||||
path_(std::move(socket_path)) {}
|
||||
|
||||
UdsServer::~UdsServer() {
|
||||
for (auto& [fd, c] : conns_) {
|
||||
@@ -201,12 +203,82 @@ void UdsServer::handle_line(Conn& c, const std::string& line) {
|
||||
}
|
||||
}
|
||||
|
||||
if (method == "download.probe") {
|
||||
handle_download_probe(c, req);
|
||||
return;
|
||||
}
|
||||
if (method == "download.refreshUrl") {
|
||||
handle_download_refreshUrl(c, req);
|
||||
return;
|
||||
}
|
||||
|
||||
// Everything else: the generated router. It returns a null json for a notification
|
||||
// that needs no reply.
|
||||
json reply = proto::dispatch(dispatcher_, proto::Transport::Uds, req);
|
||||
if (!reply.is_null()) queue_reply(c, reply);
|
||||
}
|
||||
|
||||
void UdsServer::handle_download_probe(Conn& c, const json& request) {
|
||||
const json id = request.contains("id") ? request.at("id") : json(nullptr);
|
||||
const json params_json = request.contains("params") ? request.at("params") : json::object();
|
||||
|
||||
auto parsed = proto::parse<proto::DownloadProbeParams>(params_json, "params");
|
||||
if (!parsed) {
|
||||
queue_reply(c, rpc_error(id, proto::ErrorCode::InvalidParams, parsed.error().message,
|
||||
json{{"path", parsed.error().path}}));
|
||||
return;
|
||||
}
|
||||
if (!actions_) {
|
||||
queue_reply(c, rpc_error(id, proto::ErrorCode::InternalError,
|
||||
"not implemented in this build: download.probe"));
|
||||
return;
|
||||
}
|
||||
|
||||
const int fd = c.fd;
|
||||
actions_->probe_now(
|
||||
*parsed, [this, fd, id](proto::HandlerResult<proto::DownloadProbeResult> r) {
|
||||
auto it = conns_.find(fd);
|
||||
if (it == conns_.end()) return; // client gone while the probe was outstanding
|
||||
if (r) {
|
||||
queue_reply(*it->second, proto::make_result(id, *r));
|
||||
} else {
|
||||
queue_reply(*it->second,
|
||||
rpc_error(id, r.error().code, r.error().message, r.error().data));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void UdsServer::handle_download_refreshUrl(Conn& c, const json& request) {
|
||||
const json id = request.contains("id") ? request.at("id") : json(nullptr);
|
||||
const json params_json = request.contains("params") ? request.at("params") : json::object();
|
||||
|
||||
auto parsed = proto::parse<proto::DownloadRefreshUrlParams>(params_json, "params");
|
||||
if (!parsed) {
|
||||
queue_reply(c, rpc_error(id, proto::ErrorCode::InvalidParams, parsed.error().message,
|
||||
json{{"path", parsed.error().path}}));
|
||||
return;
|
||||
}
|
||||
if (!actions_) {
|
||||
queue_reply(c, rpc_error(id, proto::ErrorCode::InternalError,
|
||||
"not implemented in this build: download.refreshUrl"));
|
||||
return;
|
||||
}
|
||||
|
||||
const int fd = c.fd;
|
||||
actions_->refresh_url(
|
||||
parsed->taskId, parsed->url, parsed->headers, parsed->cookies,
|
||||
[this, fd, id](proto::HandlerResult<proto::DownloadRefreshUrlResult> r) {
|
||||
auto it = conns_.find(fd);
|
||||
if (it == conns_.end()) return; // client gone while the probe was outstanding
|
||||
if (r) {
|
||||
queue_reply(*it->second, proto::make_result(id, *r));
|
||||
} else {
|
||||
queue_reply(*it->second,
|
||||
rpc_error(id, r.error().code, r.error().message, r.error().data));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
bool UdsServer::handle_session_method(Conn& c, const std::string& method, const json& request,
|
||||
json& reply) {
|
||||
const json id = request.contains("id") ? request.at("id") : json(nullptr);
|
||||
@@ -251,11 +323,21 @@ bool UdsServer::handle_session_method(Conn& c, const std::string& method, const
|
||||
json{{"path", p.error().path}});
|
||||
return true;
|
||||
}
|
||||
// Event fan-out is not wired yet; accept the subscription and echo it back so a
|
||||
// client can already register its interest without erroring.
|
||||
if (!c.sub_id) {
|
||||
const int fd = c.fd;
|
||||
c.sub_id = hub_.subscribe([this, fd](const json& n) {
|
||||
if (const auto it = conns_.find(fd); it != conns_.end()) queue_reply(*it->second, n);
|
||||
});
|
||||
}
|
||||
std::vector<proto::Event> events;
|
||||
proto::SessionSubscribeResult r;
|
||||
r.ok = true;
|
||||
for (const auto& ev : p->events) r.events.emplace_back(proto::to_string(ev));
|
||||
for (const auto& ev : p->events) {
|
||||
const auto name = proto::to_string(ev);
|
||||
r.events.emplace_back(name);
|
||||
if (auto e = proto::event_from_string(name)) events.push_back(*e);
|
||||
}
|
||||
hub_.set_filter(*c.sub_id, std::move(events), p->taskIds);
|
||||
reply = proto::make_result(id, r);
|
||||
return true;
|
||||
}
|
||||
@@ -300,6 +382,7 @@ void UdsServer::flush(Conn& c) {
|
||||
|
||||
void UdsServer::close_conn(int fd) {
|
||||
if (const auto it = conns_.find(fd); it != conns_.end()) {
|
||||
if (it->second->sub_id) hub_.unsubscribe(*it->second->sub_id);
|
||||
loop_.del_fd(fd);
|
||||
::close(fd);
|
||||
conns_.erase(it);
|
||||
|
||||
@@ -13,13 +13,16 @@
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <optional>
|
||||
#include <system_error>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/ndjson.hpp"
|
||||
#include "rpc/task_action_port.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
@@ -28,7 +31,11 @@ class EventLoop;
|
||||
|
||||
class UdsServer {
|
||||
public:
|
||||
UdsServer(EventLoop& loop, velox::proto::Dispatcher& dispatcher, std::string socket_path);
|
||||
// `actions` is optional (nullptr in tests that don't need download.probe) — see
|
||||
// handle_download_probe's own comment for why this method can't go through the
|
||||
// generic dispatch() path like everything else.
|
||||
UdsServer(EventLoop& loop, velox::proto::Dispatcher& dispatcher, EventHub& hub,
|
||||
std::string socket_path, TaskActionPort* actions = nullptr);
|
||||
~UdsServer();
|
||||
|
||||
UdsServer(const UdsServer&) = delete;
|
||||
@@ -51,6 +58,7 @@ private:
|
||||
bool close_after_flush = false;
|
||||
bool hello_ok = false;
|
||||
std::string session_id;
|
||||
std::optional<EventHub::SubId> sub_id;
|
||||
};
|
||||
|
||||
void on_listener_readable();
|
||||
@@ -62,12 +70,24 @@ private:
|
||||
bool handle_session_method(Conn& c, const std::string& method, const nlohmann::json& request,
|
||||
nlohmann::json& reply);
|
||||
|
||||
// download.probe is genuinely async (up to the schema's 30s x-deadlineMs, on the
|
||||
// engine's probe pool) and so cannot fit the synchronous generic dispatch() path —
|
||||
// special-cased here exactly the way handle_session_method special-cases session.*.
|
||||
// Queues the reply itself, later, when actions_->probe_now()'s callback fires; does
|
||||
// nothing if the connection is gone by then (client disconnected mid-probe).
|
||||
void handle_download_probe(Conn& c, const nlohmann::json& request);
|
||||
// Same reasoning as handle_download_probe — a real network round trip, same 30s
|
||||
// x-deadlineMs.
|
||||
void handle_download_refreshUrl(Conn& c, const nlohmann::json& request);
|
||||
|
||||
void queue_reply(Conn& c, const nlohmann::json& reply);
|
||||
void flush(Conn& c);
|
||||
void close_conn(int fd);
|
||||
|
||||
EventLoop& loop_;
|
||||
velox::proto::Dispatcher& dispatcher_;
|
||||
EventHub& hub_;
|
||||
TaskActionPort* actions_;
|
||||
std::string path_;
|
||||
int listen_fd_ = -1;
|
||||
bool bound_ = false; // path_ is ours to unlink on destruction
|
||||
|
||||
@@ -60,11 +60,14 @@ json rpc_error(const json& id, proto::ErrorCode code, std::string_view msg, json
|
||||
} // namespace
|
||||
|
||||
WsServer::WsServer(EventLoop& loop, proto::Dispatcher& dispatcher, store::Db& db,
|
||||
PairingApprover& approver, RuntimeDir runtime)
|
||||
PairingApprover& approver, EventHub& hub, RuntimeDir runtime,
|
||||
TaskActionPort* actions)
|
||||
: loop_(loop),
|
||||
dispatcher_(dispatcher),
|
||||
db_(db),
|
||||
approver_(approver),
|
||||
hub_(hub),
|
||||
actions_(actions),
|
||||
runtime_(std::move(runtime)) {}
|
||||
|
||||
WsServer::~WsServer() {
|
||||
@@ -260,10 +263,80 @@ void WsServer::handle_rpc(Conn& c, const std::string& text) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (method == "download.probe") {
|
||||
handle_download_probe(c, req);
|
||||
return;
|
||||
}
|
||||
if (method == "download.refreshUrl") {
|
||||
handle_download_refreshUrl(c, req);
|
||||
return;
|
||||
}
|
||||
|
||||
json reply = proto::dispatch(dispatcher_, proto::Transport::Ws, req);
|
||||
if (!reply.is_null()) send_text(c, reply);
|
||||
}
|
||||
|
||||
void WsServer::handle_download_probe(Conn& c, const json& request) {
|
||||
const json id = request.contains("id") ? request.at("id") : json(nullptr);
|
||||
const json params_json = request.contains("params") ? request.at("params") : json::object();
|
||||
|
||||
auto parsed = proto::parse<proto::DownloadProbeParams>(params_json, "params");
|
||||
if (!parsed) {
|
||||
send_text(c, rpc_error(id, proto::ErrorCode::InvalidParams, parsed.error().message,
|
||||
json{{"path", parsed.error().path}}));
|
||||
return;
|
||||
}
|
||||
if (!actions_) {
|
||||
send_text(c, rpc_error(id, proto::ErrorCode::InternalError,
|
||||
"not implemented in this build: download.probe"));
|
||||
return;
|
||||
}
|
||||
|
||||
const int fd = c.fd;
|
||||
actions_->probe_now(
|
||||
*parsed, [this, fd, id](proto::HandlerResult<proto::DownloadProbeResult> r) {
|
||||
auto it = conns_.find(fd);
|
||||
if (it == conns_.end()) return; // client gone while the probe was outstanding
|
||||
if (r) {
|
||||
send_text(*it->second, proto::make_result(id, *r));
|
||||
} else {
|
||||
send_text(*it->second,
|
||||
rpc_error(id, r.error().code, r.error().message, r.error().data));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void WsServer::handle_download_refreshUrl(Conn& c, const json& request) {
|
||||
const json id = request.contains("id") ? request.at("id") : json(nullptr);
|
||||
const json params_json = request.contains("params") ? request.at("params") : json::object();
|
||||
|
||||
auto parsed = proto::parse<proto::DownloadRefreshUrlParams>(params_json, "params");
|
||||
if (!parsed) {
|
||||
send_text(c, rpc_error(id, proto::ErrorCode::InvalidParams, parsed.error().message,
|
||||
json{{"path", parsed.error().path}}));
|
||||
return;
|
||||
}
|
||||
if (!actions_) {
|
||||
send_text(c, rpc_error(id, proto::ErrorCode::InternalError,
|
||||
"not implemented in this build: download.refreshUrl"));
|
||||
return;
|
||||
}
|
||||
|
||||
const int fd = c.fd;
|
||||
actions_->refresh_url(
|
||||
parsed->taskId, parsed->url, parsed->headers, parsed->cookies,
|
||||
[this, fd, id](proto::HandlerResult<proto::DownloadRefreshUrlResult> r) {
|
||||
auto it = conns_.find(fd);
|
||||
if (it == conns_.end()) return; // client gone while the probe was outstanding
|
||||
if (r) {
|
||||
send_text(*it->second, proto::make_result(id, *r));
|
||||
} else {
|
||||
send_text(*it->second,
|
||||
rpc_error(id, r.error().code, r.error().message, r.error().data));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
bool WsServer::handle_session_ws(Conn& c, const std::string& method, const json& request,
|
||||
json& reply) {
|
||||
const json id = request.contains("id") ? request.at("id") : json(nullptr);
|
||||
@@ -364,9 +437,21 @@ bool WsServer::handle_session_ws(Conn& c, const std::string& method, const json&
|
||||
json{{"path", p.error().path}});
|
||||
return true;
|
||||
}
|
||||
if (!c.sub_id) {
|
||||
const int fd = c.fd;
|
||||
c.sub_id = hub_.subscribe([this, fd](const json& n) {
|
||||
if (const auto it = conns_.find(fd); it != conns_.end()) send_text(*it->second, n);
|
||||
});
|
||||
}
|
||||
std::vector<proto::Event> events;
|
||||
proto::SessionSubscribeResult r;
|
||||
r.ok = true;
|
||||
for (const auto& ev : p->events) r.events.emplace_back(proto::to_string(ev));
|
||||
for (const auto& ev : p->events) {
|
||||
const auto name = proto::to_string(ev);
|
||||
r.events.emplace_back(name);
|
||||
if (auto e = proto::event_from_string(name)) events.push_back(*e);
|
||||
}
|
||||
hub_.set_filter(*c.sub_id, std::move(events), p->taskIds);
|
||||
reply = proto::make_result(id, r);
|
||||
return true;
|
||||
}
|
||||
@@ -423,6 +508,7 @@ void WsServer::flush(Conn& c) {
|
||||
|
||||
void WsServer::close_conn(int fd) {
|
||||
if (const auto it = conns_.find(fd); it != conns_.end()) {
|
||||
if (it->second->sub_id) hub_.unsubscribe(*it->second->sub_id);
|
||||
loop_.del_fd(fd);
|
||||
::close(fd);
|
||||
conns_.erase(it);
|
||||
|
||||
@@ -8,14 +8,17 @@
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <system_error>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/pairing.hpp"
|
||||
#include "rpc/runtime_dir.hpp"
|
||||
#include "rpc/task_action_port.hpp"
|
||||
#include "rpc/ws_frame.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
@@ -29,8 +32,12 @@ class EventLoop;
|
||||
|
||||
class WsServer {
|
||||
public:
|
||||
// `actions` is optional (nullptr in tests that don't need download.probe) — see
|
||||
// handle_download_probe's own comment for why this method can't go through the
|
||||
// generic dispatch() path like everything else.
|
||||
WsServer(EventLoop& loop, velox::proto::Dispatcher& dispatcher, velox::daemon::store::Db& db,
|
||||
PairingApprover& approver, RuntimeDir runtime);
|
||||
PairingApprover& approver, EventHub& hub, RuntimeDir runtime,
|
||||
TaskActionPort* actions = nullptr);
|
||||
~WsServer();
|
||||
|
||||
WsServer(const WsServer&) = delete;
|
||||
@@ -60,6 +67,7 @@ private:
|
||||
bool authed = false;
|
||||
std::string pairing_id;
|
||||
std::string session_id;
|
||||
std::optional<EventHub::SubId> sub_id;
|
||||
};
|
||||
|
||||
void on_listener_readable();
|
||||
@@ -70,6 +78,11 @@ private:
|
||||
bool handle_session_ws(Conn& c, const std::string& method, const nlohmann::json& request,
|
||||
nlohmann::json& reply);
|
||||
|
||||
// See UdsServer::handle_download_probe — same reasoning, same pattern, duplicated per
|
||||
// transport because each owns its own Conn/send mechanics.
|
||||
void handle_download_probe(Conn& c, const nlohmann::json& request);
|
||||
void handle_download_refreshUrl(Conn& c, const nlohmann::json& request);
|
||||
|
||||
void send_text(Conn& c, const nlohmann::json& value);
|
||||
void send_frame(Conn& c, WsOpcode op, std::string_view payload);
|
||||
void begin_close(Conn& c, std::uint16_t code, std::string_view reason);
|
||||
@@ -80,6 +93,8 @@ private:
|
||||
velox::proto::Dispatcher& dispatcher_;
|
||||
velox::daemon::store::Db& db_;
|
||||
PairingApprover& approver_;
|
||||
EventHub& hub_;
|
||||
TaskActionPort* actions_;
|
||||
RuntimeDir runtime_;
|
||||
PairingRateLimiter rate_limiter_;
|
||||
|
||||
|
||||
@@ -7,18 +7,22 @@
|
||||
// wraps `vdm::Engine` + `vdm::segment::SegmentBudget`; FakeEnginePort records calls.
|
||||
//
|
||||
// Task ids here are `vdm::TaskId` — the engine assigns one from start() and the Scheduler
|
||||
// keeps the wire-UUID <-> TaskId map (ADR 0013). Admission is the Scheduler's: it calls
|
||||
// start() only for a task the governor admitted, and the engine begins probing at once
|
||||
// (it does not queue). The min-1 fairness rule in SegmentBudget then guarantees each
|
||||
// started task a slot; set_task_order pushes the priority.
|
||||
// keeps the wire-UUID <-> TaskId map (ADR 0013). Admission is the Scheduler's: for a task
|
||||
// the governor admits, it probes first (persisting sizeBytes/resumable/validator before a
|
||||
// single byte moves), then calls start() with that ProbeResult as probe_hint. The min-1
|
||||
// fairness rule in SegmentBudget then guarantees each started task a slot; set_task_order
|
||||
// pushes the priority.
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "vdm/ids.hpp"
|
||||
#include "vdm/net/probe.hpp"
|
||||
#include "vdm/task/download.hpp"
|
||||
#include "vdm/util/result.hpp"
|
||||
|
||||
namespace velox::daemon::sched {
|
||||
|
||||
@@ -26,6 +30,13 @@ class EnginePort {
|
||||
public:
|
||||
virtual ~EnginePort() = default;
|
||||
|
||||
// Runs on the probe pool, outside the segment budget (ADR 0011 §5); `done` arrives on
|
||||
// an engine thread, exactly once. The Scheduler probes before every start() so it
|
||||
// always has a real ProbeResult (size, resumable, validator) to persist and to pass
|
||||
// back as DownloadSpec.probe_hint — one code path instead of "sometimes has one".
|
||||
virtual void probe(const vdm::net::ProbeRequest& req,
|
||||
std::function<void(vdm::Result<vdm::net::ProbeResult>)> done) = 0;
|
||||
|
||||
virtual vdm::TaskId start(const vdm::task::DownloadSpec& spec,
|
||||
vdm::task::DownloadCallbacks callbacks) = 0;
|
||||
|
||||
@@ -35,15 +46,26 @@ public:
|
||||
virtual void provide_auth(vdm::TaskId, const std::string& username,
|
||||
const std::string& password, bool remember) = 0;
|
||||
virtual void decide(vdm::TaskId, vdm::task::Decision) = 0;
|
||||
virtual void refresh_url(vdm::TaskId, const std::string& url) = 0;
|
||||
virtual void refresh_url(vdm::TaskId, const std::string& url,
|
||||
const std::vector<vdm::net::HeaderField>& headers = {}) = 0;
|
||||
|
||||
// The daemon is done with this task (it went terminal). Drop the handle. Idempotent.
|
||||
virtual void release(vdm::TaskId) = 0;
|
||||
|
||||
// A synchronous, lock-guarded snapshot (DownloadHandle::progress()). nullopt if the
|
||||
// id is unknown (already released, or never started).
|
||||
virtual std::optional<vdm::task::Progress> progress(vdm::TaskId) const = 0;
|
||||
|
||||
// ADR 0011 admission surface. Values are DAEMON's; enforcement is the engine's.
|
||||
virtual void set_task_order(const std::vector<vdm::TaskId>& order) = 0;
|
||||
virtual void set_max_active_segments(std::uint32_t n) = 0;
|
||||
virtual void set_host_segment_cap(const std::string& host, std::uint32_t cap) = 0;
|
||||
|
||||
// limiter.set: 0 means unlimited (vdm::rate::TokenBucket's own convention), applied
|
||||
// across every active transfer immediately — there is no "next task only" variant for
|
||||
// a single shared global bucket, so `applyToRunning` on the wire has nothing to select
|
||||
// between; it is accepted for schema compliance and always behaves as if true.
|
||||
virtual void set_global_speed_limit(std::uint64_t bps) = 0;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::sched
|
||||
|
||||
@@ -16,6 +16,11 @@ class EnginePortCore final : public EnginePort {
|
||||
public:
|
||||
explicit EnginePortCore(vdm::Engine& engine) : engine_(engine) {}
|
||||
|
||||
void probe(const vdm::net::ProbeRequest& req,
|
||||
std::function<void(vdm::Result<vdm::net::ProbeResult>)> done) override {
|
||||
engine_.probe(req, std::move(done));
|
||||
}
|
||||
|
||||
vdm::TaskId start(const vdm::task::DownloadSpec& spec,
|
||||
vdm::task::DownloadCallbacks callbacks) override {
|
||||
vdm::task::DownloadHandle h = engine_.start(spec, std::move(callbacks));
|
||||
@@ -40,10 +45,16 @@ public:
|
||||
void decide(vdm::TaskId id, vdm::task::Decision d) override {
|
||||
if (auto* h = find(id)) h->decide(d);
|
||||
}
|
||||
void refresh_url(vdm::TaskId id, const std::string& url) override {
|
||||
if (auto* h = find(id)) h->refresh_url(url);
|
||||
void refresh_url(vdm::TaskId id, const std::string& url,
|
||||
const std::vector<vdm::net::HeaderField>& headers) override {
|
||||
if (auto* h = find(id)) h->refresh_url(url, headers);
|
||||
}
|
||||
void release(vdm::TaskId id) override { handles_.erase(id); }
|
||||
std::optional<vdm::task::Progress> progress(vdm::TaskId id) const override {
|
||||
auto it = handles_.find(id);
|
||||
if (it == handles_.end()) return std::nullopt;
|
||||
return it->second.progress();
|
||||
}
|
||||
|
||||
void set_task_order(const std::vector<vdm::TaskId>& order) override {
|
||||
engine_.segment_budget().set_task_order(order);
|
||||
@@ -54,6 +65,9 @@ public:
|
||||
void set_host_segment_cap(const std::string& host, std::uint32_t cap) override {
|
||||
engine_.segment_budget().set_host_segment_cap(host, cap);
|
||||
}
|
||||
void set_global_speed_limit(std::uint64_t bps) override {
|
||||
engine_.rate_limiter().set_global_limit(bps);
|
||||
}
|
||||
|
||||
private:
|
||||
vdm::task::DownloadHandle* find(vdm::TaskId id) {
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "sched/engine_port.hpp"
|
||||
@@ -29,6 +30,26 @@ public:
|
||||
std::vector<std::uint32_t> max_active_segments;
|
||||
std::vector<std::pair<std::string, std::uint32_t>> host_caps;
|
||||
|
||||
// probe(): synchronous by default (a default-constructed ProbeResult — success,
|
||||
// resumable=false, no known size) so a test that doesn't care about probe details
|
||||
// still sees start() happen within the same tick(). Set auto_probe_result to nullopt
|
||||
// to switch to manual mode: probe() then just records the request and stashes `done`
|
||||
// in pending_probes for the test to resolve explicitly, in order.
|
||||
std::optional<vdm::Result<vdm::net::ProbeResult>> auto_probe_result =
|
||||
vdm::Result<vdm::net::ProbeResult>{vdm::net::ProbeResult{}};
|
||||
std::vector<vdm::net::ProbeRequest> probe_requests;
|
||||
std::vector<std::function<void(vdm::Result<vdm::net::ProbeResult>)>> pending_probes;
|
||||
|
||||
void probe(const vdm::net::ProbeRequest& req,
|
||||
std::function<void(vdm::Result<vdm::net::ProbeResult>)> done) override {
|
||||
probe_requests.push_back(req);
|
||||
if (auto_probe_result) {
|
||||
done(*auto_probe_result);
|
||||
} else {
|
||||
pending_probes.push_back(std::move(done));
|
||||
}
|
||||
}
|
||||
|
||||
vdm::TaskId start(const vdm::task::DownloadSpec& spec,
|
||||
vdm::task::DownloadCallbacks callbacks) override {
|
||||
const vdm::TaskId id{next_++};
|
||||
@@ -40,13 +61,32 @@ public:
|
||||
void cancel(vdm::TaskId id, bool discard) override { cancelled.emplace_back(id, discard); }
|
||||
void provide_auth(vdm::TaskId, const std::string&, const std::string&, bool) override {}
|
||||
void decide(vdm::TaskId, vdm::task::Decision) override {}
|
||||
void refresh_url(vdm::TaskId, const std::string&) override {}
|
||||
void refresh_url(vdm::TaskId id, const std::string& url,
|
||||
const std::vector<vdm::net::HeaderField>& headers) override {
|
||||
refreshed_urls.emplace_back(id, url, headers);
|
||||
}
|
||||
struct RefreshCall {
|
||||
vdm::TaskId id;
|
||||
std::string url;
|
||||
std::vector<vdm::net::HeaderField> headers;
|
||||
};
|
||||
std::vector<RefreshCall> refreshed_urls;
|
||||
void release(vdm::TaskId id) override { released.push_back(id); }
|
||||
std::optional<vdm::task::Progress> progress(vdm::TaskId id) const override {
|
||||
auto it = fake_progress.find(id.value);
|
||||
return it == fake_progress.end() ? std::nullopt : std::optional(it->second);
|
||||
}
|
||||
|
||||
// Tests set this to control what progress(id) returns.
|
||||
std::unordered_map<std::uint64_t, vdm::task::Progress> fake_progress;
|
||||
void set_task_order(const std::vector<vdm::TaskId>& order) override { orders.push_back(order); }
|
||||
void set_max_active_segments(std::uint32_t n) override { max_active_segments.push_back(n); }
|
||||
void set_host_segment_cap(const std::string& h, std::uint32_t c) override {
|
||||
host_caps.emplace_back(h, c);
|
||||
}
|
||||
void set_global_speed_limit(std::uint64_t bps) override { global_speed_limits.push_back(bps); }
|
||||
|
||||
std::vector<std::uint64_t> global_speed_limits;
|
||||
|
||||
const std::vector<vdm::TaskId>& last_order() const { return orders.back(); }
|
||||
|
||||
|
||||
+450
-47
@@ -6,6 +6,8 @@
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "sched/schedule_window.hpp"
|
||||
#include "store/categories.hpp"
|
||||
#include "store/segments.hpp"
|
||||
#include "store/settings.hpp"
|
||||
#include "store/tasks.hpp"
|
||||
|
||||
@@ -100,10 +102,33 @@ std::vector<proto::TaskState> non_terminal_states() {
|
||||
proto::TaskState::Verifying};
|
||||
}
|
||||
|
||||
TaskErrorFields to_error_fields(const vdm::ErrorInfo& err) {
|
||||
TaskErrorFields ef;
|
||||
ef.code = std::string(vdm::error_name(err.code)); // matches TaskErrorCode by name (ADR 0010)
|
||||
ef.message = err.context;
|
||||
if (err.http_status != 0) ef.http_status = err.http_status;
|
||||
ef.retryable = err.retryable;
|
||||
return ef;
|
||||
}
|
||||
|
||||
std::string segment_state_name(vdm::segment::SegState s) {
|
||||
using S = vdm::segment::SegState;
|
||||
switch (s) {
|
||||
case S::idle: return "pending";
|
||||
case S::connecting: return "connecting";
|
||||
case S::downloading: return "downloading";
|
||||
case S::stalled: return "stalled";
|
||||
case S::complete: return "complete";
|
||||
case S::failed: return "failed";
|
||||
}
|
||||
return "pending";
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Scheduler::Scheduler(store::Db& db, EnginePort& engine, Governor governor, Deps deps)
|
||||
: db_(db), engine_(engine), governor_(std::move(governor)), deps_(std::move(deps)) {
|
||||
Scheduler::Scheduler(store::Db& db, EnginePort& engine, Governor governor, rpc::EventHub* hub,
|
||||
Deps deps)
|
||||
: db_(db), engine_(engine), governor_(std::move(governor)), hub_(hub), deps_(std::move(deps)) {
|
||||
if (!deps_.local_now) deps_.local_now = local_now_default;
|
||||
if (!deps_.post_to_loop) deps_.post_to_loop = [](std::function<void()> f) { f(); };
|
||||
}
|
||||
@@ -158,6 +183,16 @@ store::DbResult<void> Scheduler::reload_config() {
|
||||
governor_.set_config(cfg);
|
||||
engine_.set_max_active_segments(
|
||||
static_cast<std::uint32_t>(std::max<std::int64_t>(cfg.max_active_segments, 1)));
|
||||
|
||||
// The global speed limit persists across a restart the same as any other setting, but
|
||||
// (unlike connection.* above) nothing re-derives it into engine state on its own —
|
||||
// limiter.set is the only other place that calls set_global_speed_limit, and that only
|
||||
// fires on an explicit RPC in a running daemon. Push it here too so a limit set in a
|
||||
// previous run is not silently unlimited again after a restart.
|
||||
const bool limit_enabled = settings.get_bool("downloads.speedLimitEnabled");
|
||||
const std::int64_t limit_bps = settings.get_int("downloads.speedLimitBps");
|
||||
engine_.set_global_speed_limit(limit_enabled ? static_cast<std::uint64_t>(std::max<std::int64_t>(limit_bps, 0))
|
||||
: 0);
|
||||
return {};
|
||||
}
|
||||
|
||||
@@ -211,56 +246,32 @@ store::DbResult<void> Scheduler::tick() {
|
||||
const Decision d = governor_.evaluate(views, queues);
|
||||
|
||||
// --- apply ------------------------------------------------------------------
|
||||
// to_start: probe first, always — a real ProbeResult (size, resumable, validator) is
|
||||
// what makes sizeBytes/resumable correct on the wire, not a post-hoc guess. The task
|
||||
// moves to `probing` immediately so the governor does not re-admit it on the next
|
||||
// tick while the (possibly slow, always async) probe is outstanding.
|
||||
for (const auto& wire_id : d.to_start) {
|
||||
auto got = tasks.get(wire_id);
|
||||
if (!got || !got->has_value()) continue;
|
||||
const store::TaskRow& row = **got;
|
||||
|
||||
vdm::task::DownloadSpec spec;
|
||||
spec.url = row.url;
|
||||
spec.save_path = row.save_dir + "/" + row.filename;
|
||||
if (row.req_segments) spec.segments = static_cast<std::uint32_t>(*row.req_segments);
|
||||
if (row.req_buffer_bytes)
|
||||
spec.buffer_bytes = static_cast<std::uint64_t>(*row.req_buffer_bytes);
|
||||
if (row.checksum_algo && row.checksum_value) {
|
||||
vdm::task::Checksum ck;
|
||||
ck.hex = *row.checksum_value;
|
||||
if (*row.checksum_algo == "md5") ck.algo = vdm::task::Checksum::Algo::md5;
|
||||
else if (*row.checksum_algo == "sha1") ck.algo = vdm::task::Checksum::Algo::sha1;
|
||||
else if (*row.checksum_algo == "sha512") ck.algo = vdm::task::Checksum::Algo::sha512;
|
||||
else ck.algo = vdm::task::Checksum::Algo::sha256;
|
||||
spec.checksum = ck;
|
||||
}
|
||||
spec.allow_resume = true; // resume from a sidecar if one is beside save_path
|
||||
transition(wire_id, "probing", std::nullopt, std::nullopt);
|
||||
|
||||
vdm::net::ProbeRequest req;
|
||||
req.url = row.url;
|
||||
// headers / cookies / referrer / user_agent are not persisted yet (a URL-only
|
||||
// `velox add` has none); the capture path will fill them when it lands.
|
||||
|
||||
vdm::task::DownloadCallbacks cbs;
|
||||
const std::string id_copy = wire_id;
|
||||
cbs.on_state = [this, id_copy](vdm::task::EngineState, vdm::task::EngineState to,
|
||||
const std::optional<vdm::ErrorInfo>& err) {
|
||||
std::optional<TaskErrorFields> ef;
|
||||
if (err) {
|
||||
ef = TaskErrorFields{};
|
||||
ef->code = std::string(vdm::error_name(err->code)); // matches TaskErrorCode
|
||||
ef->message = err->context;
|
||||
if (err->http_status != 0) ef->http_status = err->http_status;
|
||||
ef->retryable = err->retryable;
|
||||
}
|
||||
const std::string to_name = engine_state_name(to);
|
||||
deps_.post_to_loop(
|
||||
[this, id_copy, to_name, ef]() { on_engine_state(id_copy, to_name, ef); });
|
||||
};
|
||||
|
||||
const vdm::TaskId engine_id = engine_.start(spec, std::move(cbs));
|
||||
map(wire_id, engine_id);
|
||||
(void)tasks.set_state(wire_id, "probing", std::nullopt);
|
||||
engine_.probe(req, [this, id_copy](vdm::Result<vdm::net::ProbeResult> pr) {
|
||||
deps_.post_to_loop([this, id_copy, pr]() { on_probe_result(id_copy, pr); });
|
||||
});
|
||||
}
|
||||
|
||||
for (const auto& wire_id : d.to_resume) {
|
||||
if (auto eid = engine_id_of(wire_id)) {
|
||||
engine_.resume(*eid);
|
||||
(void)tasks.set_state(wire_id, "connecting", std::nullopt);
|
||||
transition(wire_id, "connecting", std::nullopt, std::nullopt);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,7 +280,7 @@ store::DbResult<void> Scheduler::tick() {
|
||||
? pause_reason_str(d.pause_reasons.at(wire_id))
|
||||
: "user";
|
||||
if (auto eid = engine_id_of(wire_id)) engine_.pause(*eid);
|
||||
(void)tasks.set_state(wire_id, "paused", std::string(reason));
|
||||
transition(wire_id, "paused", std::string(reason), std::nullopt);
|
||||
}
|
||||
|
||||
std::vector<vdm::TaskId> order;
|
||||
@@ -281,14 +292,27 @@ store::DbResult<void> Scheduler::tick() {
|
||||
return {};
|
||||
}
|
||||
|
||||
void Scheduler::on_engine_state(const std::string& wire_id, std::string_view engine_state,
|
||||
const std::optional<TaskErrorFields>& err) {
|
||||
void Scheduler::transition(const std::string& wire_id, std::string_view to_state,
|
||||
std::optional<std::string> pause_reason,
|
||||
const std::optional<TaskErrorFields>& err) {
|
||||
store::Tasks tasks(db_);
|
||||
// pause_reason: an engine-initiated pause carries an error => 'auto' (ADR 0013 §2);
|
||||
// otherwise set_state clears the column.
|
||||
std::optional<std::string> reason;
|
||||
if (engine_state == "paused" && err) reason = "auto";
|
||||
(void)tasks.set_state(wire_id, engine_state, reason);
|
||||
const auto before = tasks.get(wire_id);
|
||||
const std::string previous = (before && before->has_value()) ? (**before).state : std::string();
|
||||
|
||||
// An engine-initiated pause carries an error => 'auto' (ADR 0013 §2), overriding
|
||||
// whatever the caller passed (a scheduler-driven pause never carries an error here).
|
||||
std::optional<std::string> reason = pause_reason;
|
||||
if (to_state == "paused" && err) {
|
||||
reason = "auto";
|
||||
} else if (to_state == "paused" && !reason && before && before->has_value()) {
|
||||
// No reason supplied — the common case is the engine's own pause-ack callback
|
||||
// (on_state(_, paused, nullopt)) arriving after whoever actually initiated the
|
||||
// pause (user_pause() or tick()'s to_pause loop) already wrote the real reason
|
||||
// eagerly. Keep what's already stored instead of clobbering it back to NULL:
|
||||
// set_state() always overwrites the column, reason or not.
|
||||
reason = (**before).pause_reason;
|
||||
}
|
||||
(void)tasks.set_state(wire_id, to_state, reason);
|
||||
|
||||
if (err) {
|
||||
auto st = db_.prepare(
|
||||
@@ -306,12 +330,391 @@ void Scheduler::on_engine_state(const std::string& wire_id, std::string_view eng
|
||||
}
|
||||
}
|
||||
|
||||
if (engine_state == "complete" || engine_state == "failed" || engine_state == "cancelled") {
|
||||
if (!hub_) return;
|
||||
auto after = tasks.get(wire_id);
|
||||
if (!after || !after->has_value()) return;
|
||||
const proto::TaskSummary summary = store::to_summary(**after);
|
||||
|
||||
nlohmann::json params{
|
||||
{"taskId", wire_id},
|
||||
{"state", std::string(to_state)},
|
||||
{"previousState", previous.empty() ? nlohmann::json(nullptr) : nlohmann::json(previous)},
|
||||
{"summary", summary},
|
||||
{"error", summary.error.has_value() ? nlohmann::json(*summary.error) : nlohmann::json(nullptr)},
|
||||
};
|
||||
hub_->publish(proto::Event::TaskState, proto::make_notification(proto::Event::TaskState, params),
|
||||
wire_id);
|
||||
}
|
||||
|
||||
void Scheduler::on_engine_state(const std::string& wire_id, std::string_view from_state,
|
||||
std::string_view to_state,
|
||||
const std::optional<TaskErrorFields>& err) {
|
||||
(void)from_state; // transition() reads the store's own current state as previousState,
|
||||
// which is authoritative regardless of engine/store timing
|
||||
transition(wire_id, to_state, std::nullopt, err);
|
||||
|
||||
if (to_state == "complete" || to_state == "failed" || to_state == "cancelled") {
|
||||
if (auto eid = engine_id_of(wire_id)) {
|
||||
// One last snapshot before the handle goes away: a task that never lived past
|
||||
// a single tick (small/fast/local) would otherwise leave downloadedBytes and
|
||||
// segmentDetail at their pre-segmentation defaults forever, in violation of
|
||||
// TaskDetail.segmentDetail's "exactly summary.segments entries" contract.
|
||||
if (const auto p = engine_.progress(*eid)) persist_progress(wire_id, *p);
|
||||
engine_.release(*eid);
|
||||
unmap_engine(*eid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
vdm::task::DownloadCallbacks Scheduler::make_callbacks(const std::string& wire_id) {
|
||||
vdm::task::DownloadCallbacks cbs;
|
||||
const std::string id_copy = wire_id;
|
||||
|
||||
cbs.on_state = [this, id_copy](vdm::task::EngineState from, vdm::task::EngineState to,
|
||||
const std::optional<vdm::ErrorInfo>& err) {
|
||||
std::optional<TaskErrorFields> ef;
|
||||
if (err) ef = to_error_fields(*err);
|
||||
const std::string from_name = engine_state_name(from);
|
||||
const std::string to_name = engine_state_name(to);
|
||||
deps_.post_to_loop([this, id_copy, from_name, to_name, ef]() {
|
||||
on_engine_state(id_copy, from_name, to_name, ef);
|
||||
});
|
||||
};
|
||||
|
||||
cbs.on_finished = [this, id_copy](vdm::Result<vdm::task::DownloadOutcome> outcome) {
|
||||
deps_.post_to_loop([this, id_copy, outcome]() { on_engine_finished(id_copy, outcome); });
|
||||
};
|
||||
|
||||
return cbs;
|
||||
}
|
||||
|
||||
void Scheduler::on_probe_result(const std::string& wire_id,
|
||||
const vdm::Result<vdm::net::ProbeResult>& pr) {
|
||||
auto got = store::Tasks(db_).get(wire_id);
|
||||
if (!got || !got->has_value()) return; // removed while the probe was outstanding
|
||||
const store::TaskRow& row = **got;
|
||||
|
||||
if (!pr) {
|
||||
transition(wire_id, "failed", std::nullopt, to_error_fields(pr.error()));
|
||||
return;
|
||||
}
|
||||
|
||||
store::Tasks::ProbeFields fields;
|
||||
if (pr->total_size) fields.size_bytes = static_cast<std::int64_t>(*pr->total_size);
|
||||
fields.resumable = pr->resumable;
|
||||
if (!pr->etag.empty()) fields.etag = pr->etag;
|
||||
if (!pr->last_modified.empty()) fields.last_modified = pr->last_modified;
|
||||
if (!pr->mime.empty()) fields.content_type = pr->mime;
|
||||
if (pr->effective_url != row.url) fields.effective_url = pr->effective_url;
|
||||
(void)store::Tasks(db_).set_probe_result(wire_id, fields);
|
||||
|
||||
vdm::task::DownloadSpec spec;
|
||||
spec.url = row.url;
|
||||
spec.save_path = row.save_dir + "/" + row.filename;
|
||||
if (row.req_segments) spec.segments = static_cast<std::uint32_t>(*row.req_segments);
|
||||
if (row.req_buffer_bytes) spec.buffer_bytes = static_cast<std::uint64_t>(*row.req_buffer_bytes);
|
||||
if (row.checksum_algo && row.checksum_value) {
|
||||
vdm::task::Checksum ck;
|
||||
ck.hex = *row.checksum_value;
|
||||
if (*row.checksum_algo == "md5") ck.algo = vdm::task::Checksum::Algo::md5;
|
||||
else if (*row.checksum_algo == "sha1") ck.algo = vdm::task::Checksum::Algo::sha1;
|
||||
else if (*row.checksum_algo == "sha512") ck.algo = vdm::task::Checksum::Algo::sha512;
|
||||
else ck.algo = vdm::task::Checksum::Algo::sha256;
|
||||
spec.checksum = ck;
|
||||
}
|
||||
spec.allow_resume = true; // resume from a sidecar if one is beside save_path
|
||||
spec.probe_hint = *pr; // skip a second probe; the engine still revalidates on resume
|
||||
|
||||
const vdm::TaskId engine_id = engine_.start(spec, make_callbacks(wire_id));
|
||||
map(wire_id, engine_id);
|
||||
// State stays `probing`; the engine's own on_state (probe_hint => starts in
|
||||
// `connecting`) drives the next transition through on_engine_state.
|
||||
}
|
||||
|
||||
void Scheduler::on_engine_finished(const std::string& wire_id,
|
||||
const vdm::Result<vdm::task::DownloadOutcome>& outcome) {
|
||||
// The state transition (complete/failed/cancelled) already happened via on_state,
|
||||
// which always precedes on_finished. This only tops up the byte count for a task that
|
||||
// completed before any progress tick ran — otherwise a fast/local/small transfer
|
||||
// reports downloadedBytes: 0 forever despite a byte-correct file on disk.
|
||||
if (outcome) (void)store::Tasks(db_).set_final_bytes(wire_id, static_cast<std::int64_t>(outcome->bytes));
|
||||
}
|
||||
|
||||
void Scheduler::persist_progress(const std::string& wire_id, const vdm::task::Progress& p) {
|
||||
// TaskDetail.segmentDetail is contractually "exactly TaskSummary.segments entries" —
|
||||
// so the count that goes on the wire as `segments` has to be the length of the list
|
||||
// that actually becomes segmentDetail, not effective_segments (budget slots *held*,
|
||||
// per engine_port.hpp; a small file can hold 8 fairness slots while its segmenter
|
||||
// only ever carves 2 ranges). Falls back to effective_segments only before the task
|
||||
// has any ranges yet, so a `probing`/`connecting` task still reports a sane count.
|
||||
const std::int64_t seg_count = !p.segments.empty()
|
||||
? static_cast<std::int64_t>(p.segments.size())
|
||||
: static_cast<std::int64_t>(p.effective_segments);
|
||||
|
||||
(void)store::Tasks(db_).update_progress(wire_id, static_cast<std::int64_t>(p.downloaded),
|
||||
static_cast<std::int64_t>(p.speed_bps), seg_count,
|
||||
static_cast<std::int64_t>(p.effective_buffer_bytes));
|
||||
|
||||
if (!p.segments.empty()) {
|
||||
std::vector<store::SegmentSnapshot> snaps;
|
||||
snaps.reserve(p.segments.size());
|
||||
for (const auto& s : p.segments) {
|
||||
snaps.push_back({s.index, static_cast<std::int64_t>(s.start),
|
||||
static_cast<std::int64_t>(s.end),
|
||||
static_cast<std::int64_t>(s.completed),
|
||||
static_cast<std::int64_t>(s.speed_bps),
|
||||
segment_state_name(s.state)});
|
||||
}
|
||||
(void)store::Segments(db_).replace_all(wire_id, snaps);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<Scheduler::ProgressRow> Scheduler::progress_snapshot() {
|
||||
std::vector<ProgressRow> out;
|
||||
if (to_engine_.empty()) return out;
|
||||
|
||||
out.reserve(to_engine_.size());
|
||||
for (const auto& [wire_id, engine_id] : to_engine_) {
|
||||
const auto p = engine_.progress(engine_id);
|
||||
if (!p) continue;
|
||||
|
||||
ProgressRow row;
|
||||
row.task_id = wire_id;
|
||||
row.downloaded_bytes = p->downloaded;
|
||||
row.speed_bps = p->speed_bps;
|
||||
row.eta_seconds = p->eta_seconds;
|
||||
for (const auto& s : p->segments)
|
||||
row.segments.push_back({s.index, s.completed, s.speed_bps});
|
||||
out.push_back(std::move(row));
|
||||
|
||||
persist_progress(wire_id, *p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
namespace {
|
||||
bool is_terminal_state(const std::string& s) {
|
||||
return s == "complete" || s == "failed" || s == "cancelled";
|
||||
}
|
||||
} // namespace
|
||||
|
||||
rpc::TaskActionPort::Result Scheduler::user_pause(const std::string& wire_id) {
|
||||
store::Tasks tasks(db_);
|
||||
auto got = tasks.get(wire_id);
|
||||
if (!got || !got->has_value()) return {false, false, {}};
|
||||
const store::TaskRow& row = **got;
|
||||
if (is_terminal_state(row.state) || row.state == "paused")
|
||||
return {true, false, row.state};
|
||||
|
||||
if (auto eid = engine_id_of(wire_id)) engine_.pause(*eid);
|
||||
transition(wire_id, "paused", std::string("user"), std::nullopt);
|
||||
return {true, true, "paused"};
|
||||
}
|
||||
|
||||
rpc::TaskActionPort::Result Scheduler::user_resume(const std::string& wire_id) {
|
||||
store::Tasks tasks(db_);
|
||||
auto got = tasks.get(wire_id);
|
||||
if (!got || !got->has_value()) return {false, false, {}};
|
||||
const store::TaskRow& row = **got;
|
||||
if (row.state != "paused") return {true, false, row.state};
|
||||
|
||||
if (auto eid = engine_id_of(wire_id)) {
|
||||
engine_.resume(*eid);
|
||||
transition(wire_id, "connecting", std::nullopt, std::nullopt);
|
||||
return {true, true, "connecting"};
|
||||
}
|
||||
transition(wire_id, "queued", std::nullopt, std::nullopt);
|
||||
return {true, true, "queued"};
|
||||
}
|
||||
|
||||
rpc::TaskActionPort::Result Scheduler::user_start(const std::string& wire_id) {
|
||||
store::Tasks tasks(db_);
|
||||
auto got = tasks.get(wire_id);
|
||||
if (!got || !got->has_value()) return {false, false, {}};
|
||||
const store::TaskRow& row = **got;
|
||||
if (row.state != "paused" && row.state != "new") return {true, false, row.state};
|
||||
|
||||
if (auto eid = engine_id_of(wire_id)) {
|
||||
engine_.resume(*eid);
|
||||
transition(wire_id, "connecting", std::nullopt, std::nullopt);
|
||||
return {true, true, "connecting"};
|
||||
}
|
||||
transition(wire_id, "queued", std::nullopt, std::nullopt);
|
||||
return {true, true, "queued"};
|
||||
}
|
||||
|
||||
rpc::TaskActionPort::Result Scheduler::user_cancel(const std::string& wire_id,
|
||||
bool discard_partial) {
|
||||
store::Tasks tasks(db_);
|
||||
auto got = tasks.get(wire_id);
|
||||
if (!got || !got->has_value()) return {false, false, {}};
|
||||
const store::TaskRow& row = **got;
|
||||
if (is_terminal_state(row.state)) return {true, false, row.state};
|
||||
|
||||
// Same pattern as tick()'s to_pause loop: call the engine (async, no synchronous
|
||||
// effect) and transition the store eagerly so download.get/list are correct the
|
||||
// instant this call returns. The engine's own on_state(_, cancelled, nullopt) +
|
||||
// on_finished arrive later via on_engine_state, which is what actually
|
||||
// release()s/unmaps the handle — never done here.
|
||||
if (auto eid = engine_id_of(wire_id)) engine_.cancel(*eid, discard_partial);
|
||||
transition(wire_id, "cancelled", std::nullopt, std::nullopt);
|
||||
return {true, true, "cancelled"};
|
||||
}
|
||||
|
||||
std::vector<std::string> Scheduler::pause_queue(const std::string& queue_id) {
|
||||
store::Tasks tasks(db_);
|
||||
proto::TaskFilter filter;
|
||||
filter.queueId = queue_id;
|
||||
filter.states = non_terminal_states();
|
||||
// No paging needed: a queue's max_concurrent is <= 32, so "everything non-terminal in
|
||||
// this queue" is never a large page.
|
||||
auto page = tasks.list(filter, std::nullopt, 0, 10000);
|
||||
std::vector<std::string> paused;
|
||||
if (!page) return paused;
|
||||
|
||||
for (const auto& row : page->rows) {
|
||||
if (run_state_of(row.state) != RunState::Running) continue;
|
||||
if (auto eid = engine_id_of(row.task_id)) engine_.pause(*eid);
|
||||
transition(row.task_id, "paused", std::string("queue_stopped"), std::nullopt);
|
||||
paused.push_back(row.task_id);
|
||||
}
|
||||
return paused;
|
||||
}
|
||||
|
||||
bool Scheduler::provide_auth(const std::string& wire_id, const std::string& username,
|
||||
const std::string& password, bool remember) {
|
||||
(void)remember; // not yet wired to the Secret Service anywhere in this build
|
||||
auto eid = engine_id_of(wire_id);
|
||||
if (!eid) return false;
|
||||
engine_.provide_auth(*eid, username, password, remember);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void Scheduler::probe_now(
|
||||
const proto::DownloadProbeParams& params,
|
||||
std::function<void(proto::HandlerResult<proto::DownloadProbeResult>)> done) {
|
||||
vdm::net::ProbeRequest req;
|
||||
req.url = params.url;
|
||||
if (params.headers)
|
||||
for (const auto& [k, v] : *params.headers) req.headers.push_back({k, v});
|
||||
if (params.cookies)
|
||||
for (const auto& c : *params.cookies) req.cookies.push_back({c.name, c.value});
|
||||
if (params.referrer) req.referrer = *params.referrer;
|
||||
if (params.userAgent) req.user_agent = *params.userAgent;
|
||||
|
||||
engine_.probe(req, [this, params, done](vdm::Result<vdm::net::ProbeResult> pr) {
|
||||
deps_.post_to_loop([this, params, done, pr]() {
|
||||
if (!pr) {
|
||||
nlohmann::json data;
|
||||
if (pr.error().http_status != 0) data["httpStatus"] = pr.error().http_status;
|
||||
done(std::unexpected(proto::HandlerError{
|
||||
proto::ErrorCode::ProbeFailed, pr.error().context, data}));
|
||||
return;
|
||||
}
|
||||
|
||||
const std::string filename = vdm::net::suggest_filename(*pr);
|
||||
|
||||
proto::DownloadProbeResult r;
|
||||
r.filename = filename.empty() ? "download.bin" : filename;
|
||||
if (pr->total_size) r.sizeBytes = static_cast<std::int64_t>(*pr->total_size);
|
||||
r.mime = pr->mime;
|
||||
r.resumable = pr->resumable;
|
||||
r.effectiveUrl = pr->effective_url.empty() ? params.url : pr->effective_url;
|
||||
r.suggestedCategoryId = store::Categories(db_).guess_by_extension(r.filename);
|
||||
if (!pr->etag.empty()) r.etag = pr->etag;
|
||||
if (!pr->last_modified.empty()) r.lastModified = pr->last_modified;
|
||||
r.acceptRanges = pr->accept_ranges;
|
||||
if (!pr->redirect_chain.empty()) r.redirectChain = pr->redirect_chain;
|
||||
if (pr->requires_auth) r.requiresAuth = true;
|
||||
|
||||
store::Settings settings(db_);
|
||||
store::Categories categories(db_);
|
||||
if (auto cats = categories.list()) {
|
||||
for (const auto& c : *cats)
|
||||
if (c.categoryId == r.suggestedCategoryId) {
|
||||
r.suggestedSaveDir = c.saveDir;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!r.suggestedSaveDir) r.suggestedSaveDir = settings.get_string("saveTo.defaultDir");
|
||||
|
||||
done(r);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void Scheduler::refresh_url(
|
||||
const std::string& wire_id, const std::string& url,
|
||||
const std::optional<proto::Headers>& headers,
|
||||
const std::optional<std::vector<proto::Cookie>>& cookies,
|
||||
std::function<void(proto::HandlerResult<proto::DownloadRefreshUrlResult>)> done) {
|
||||
auto got = store::Tasks(db_).get(wire_id);
|
||||
if (!got || !got->has_value()) {
|
||||
done(std::unexpected(proto::HandlerError{proto::ErrorCode::TaskNotFound, "no such task",
|
||||
nlohmann::json{{"taskId", wire_id}}}));
|
||||
return;
|
||||
}
|
||||
const store::TaskRow row = **got; // copied: read again by the async callback below
|
||||
|
||||
vdm::net::ProbeRequest req;
|
||||
req.url = url;
|
||||
if (headers)
|
||||
for (const auto& [k, v] : *headers) req.headers.push_back({k, v});
|
||||
if (cookies)
|
||||
for (const auto& c : *cookies) req.cookies.push_back({c.name, c.value});
|
||||
|
||||
engine_.probe(req, [this, wire_id, row, url, headers, done](vdm::Result<vdm::net::ProbeResult> pr) {
|
||||
deps_.post_to_loop([this, wire_id, row, url, headers, done, pr]() {
|
||||
if (!pr) {
|
||||
nlohmann::json data;
|
||||
if (pr.error().http_status != 0) data["httpStatus"] = pr.error().http_status;
|
||||
done(std::unexpected(proto::HandlerError{proto::ErrorCode::ProbeFailed,
|
||||
pr.error().context, data}));
|
||||
return;
|
||||
}
|
||||
|
||||
// "if they do not [match], it says so rather than silently restarting" (the
|
||||
// schema's own words) — comparison only fires when both sides actually have a
|
||||
// value; an unknown size/validator on either end is not itself a mismatch.
|
||||
bool content_changed = false;
|
||||
if (row.size_bytes && pr->total_size &&
|
||||
*row.size_bytes != static_cast<std::int64_t>(*pr->total_size))
|
||||
content_changed = true;
|
||||
if (row.etag && !row.etag->empty() && !pr->etag.empty() && *row.etag != pr->etag)
|
||||
content_changed = true;
|
||||
if (row.last_modified && !row.last_modified->empty() && !pr->last_modified.empty() &&
|
||||
*row.last_modified != pr->last_modified)
|
||||
content_changed = true;
|
||||
|
||||
store::Tasks tasks(db_);
|
||||
store::Tasks::ProbeFields fields;
|
||||
if (pr->total_size) fields.size_bytes = static_cast<std::int64_t>(*pr->total_size);
|
||||
fields.resumable = pr->resumable;
|
||||
if (!pr->etag.empty()) fields.etag = pr->etag;
|
||||
if (!pr->last_modified.empty()) fields.last_modified = pr->last_modified;
|
||||
if (!pr->mime.empty()) fields.content_type = pr->mime;
|
||||
const std::string effective = pr->effective_url.empty() ? url : pr->effective_url;
|
||||
fields.effective_url = effective;
|
||||
(void)tasks.set_probe_result(wire_id, fields);
|
||||
(void)tasks.set_url(wire_id, url);
|
||||
|
||||
if (auto eid = engine_id_of(wire_id)) {
|
||||
std::vector<vdm::net::HeaderField> hdrs;
|
||||
if (headers)
|
||||
for (const auto& [k, v] : *headers) hdrs.push_back({k, v});
|
||||
engine_.refresh_url(*eid, url, hdrs);
|
||||
}
|
||||
|
||||
proto::DownloadRefreshUrlResult r;
|
||||
r.ok = true;
|
||||
r.resumable = pr->resumable;
|
||||
r.contentChanged = content_changed;
|
||||
if (pr->total_size) r.sizeBytes = static_cast<std::int64_t>(*pr->total_size);
|
||||
r.effectiveUrl = effective;
|
||||
done(r);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::sched
|
||||
|
||||
+139
-13
@@ -4,13 +4,17 @@
|
||||
// wire-UUID <-> vdm::TaskId map and is the only thing that calls EnginePort::start /
|
||||
// pause / resume / set_task_order.
|
||||
//
|
||||
// Threading: tick(), reload_config(), reconcile_after_restart() and the on_engine_*
|
||||
// callbacks all run on ONE thread (the RPC loop). Engine callbacks arrive on engine
|
||||
// threads, so the real wiring passes a `post_to_loop` that marshals them here; the
|
||||
// default runs them inline (tests, single-threaded).
|
||||
// Threading: tick(), reload_config(), reconcile_after_restart(), progress_snapshot() and
|
||||
// the on_engine_* callbacks all run on ONE thread (the RPC loop). Engine callbacks arrive
|
||||
// on engine threads, so the real wiring passes a `post_to_loop` that marshals them here;
|
||||
// the default runs them inline (tests, single-threaded).
|
||||
//
|
||||
// Not yet wired into veloxd — that plus the real EnginePort land when velox::core's
|
||||
// stage-8 bodies reach main (daemon/docs/deferrals.md D4).
|
||||
// event.task.state (D5): on_engine_state publishes it when a hub is supplied — the same
|
||||
// callback that keeps the store row current also keeps subscribed clients current.
|
||||
// event.task.progress is NOT published here: it must be batched into one array message at
|
||||
// <=4 Hz (event.task.progress.schema.json x-maxRateHz), so the caller collects
|
||||
// progress_snapshot() on its own 250 ms timer and does one hub_.publish() with the whole
|
||||
// array, never one per task.
|
||||
|
||||
#include <cstdint>
|
||||
#include <ctime>
|
||||
@@ -18,11 +22,17 @@
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/task_action_port.hpp"
|
||||
#include "sched/engine_port.hpp"
|
||||
#include "sched/governor.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "vdm/ids.hpp"
|
||||
#include "vdm/net/probe.hpp"
|
||||
#include "vdm/task/download.hpp"
|
||||
#include "vdm/util/result.hpp"
|
||||
|
||||
namespace velox::daemon::sched {
|
||||
|
||||
@@ -36,7 +46,12 @@ struct TaskErrorFields {
|
||||
std::optional<std::int64_t> attempt;
|
||||
};
|
||||
|
||||
class Scheduler {
|
||||
// Implements rpc::TaskActionPort directly — sched/ already depends on rpc/ (EventHub), so
|
||||
// this costs nothing new, and it's what lets dispatcher.hpp depend on the port interface
|
||||
// instead of on sched/scheduler.hpp (see rpc/task_action_port.hpp's top comment for why
|
||||
// that matters: it would otherwise make veloxd_rpc <-> veloxd_sched a circular library
|
||||
// dependency).
|
||||
class Scheduler final : public rpc::TaskActionPort {
|
||||
public:
|
||||
// `local_now` returns a fully-populated std::tm in local time; injected so tests can
|
||||
// pin the clock. `post_to_loop` marshals an engine-thread callback onto the loop
|
||||
@@ -46,7 +61,10 @@ public:
|
||||
std::function<void(std::function<void()>)> post_to_loop;
|
||||
};
|
||||
|
||||
Scheduler(store::Db& db, EnginePort& engine, Governor governor, Deps deps = {});
|
||||
// `hub` is optional so unit tests can build a Scheduler with no event fan-out at all;
|
||||
// production wiring always supplies one.
|
||||
Scheduler(store::Db& db, EnginePort& engine, Governor governor, rpc::EventHub* hub = nullptr,
|
||||
Deps deps = {});
|
||||
|
||||
// ADR 0013 §5: on daemon start, every task whose persisted state is a CORE-owned one
|
||||
// (probing..verifying) is rewritten to `queued`; `paused` keeps its pauseReason. The
|
||||
@@ -62,11 +80,87 @@ public:
|
||||
// (start / resume / pause via the engine, update task state rows, push set_task_order).
|
||||
store::DbResult<void> tick();
|
||||
|
||||
// Engine lifecycle callback -> store projection, so the next tick sees ground truth.
|
||||
// Keyed by wire UUID (known when the callback is built, before start() returns the
|
||||
// TaskId). Also the hook the event.task.state fan-out will use — D5.
|
||||
void on_engine_state(const std::string& wire_id, std::string_view engine_state,
|
||||
const std::optional<TaskErrorFields>& err);
|
||||
// Engine lifecycle callback -> store projection, so the next tick sees ground truth,
|
||||
// and (when a hub was supplied) the event.task.state publish. Keyed by wire UUID
|
||||
// (known when the callback is built, before start() returns the TaskId).
|
||||
void on_engine_state(const std::string& wire_id, std::string_view from_state,
|
||||
std::string_view to_state, const std::optional<TaskErrorFields>& err);
|
||||
|
||||
// One row per task the engine is currently tracking, for the caller's
|
||||
// event.task.progress batch. Also writes downloaded_bytes / eff_segments /
|
||||
// eff_buffer_bytes back to the store so download.list / download.get stay current
|
||||
// between state transitions.
|
||||
struct ProgressRow {
|
||||
std::string task_id;
|
||||
std::uint64_t downloaded_bytes;
|
||||
std::uint64_t speed_bps;
|
||||
std::optional<std::uint32_t> eta_seconds;
|
||||
struct Segment {
|
||||
std::uint32_t index;
|
||||
std::uint64_t downloaded_bytes;
|
||||
std::uint64_t speed_bps;
|
||||
};
|
||||
std::vector<Segment> segments;
|
||||
};
|
||||
std::vector<ProgressRow> progress_snapshot();
|
||||
|
||||
// rpc::TaskActionPort. These apply immediately — never wait for the next tick() —
|
||||
// because pausing, resuming or cancelling a live transfer cannot wait up to 1s for the
|
||||
// timerfd, and the governor will never do any of them on its own for a user-owned
|
||||
// reason (ADR 0013 §3: "never touch a task paused for a reason it does not own").
|
||||
// Idempotent: calling one on a task already in the target (or a terminal) state
|
||||
// reports found=true, changed=false.
|
||||
rpc::TaskActionPort::Result user_pause(const std::string& wire_id) override;
|
||||
// A task still holding a live engine handle (paused mid-flight) is engine_.resume()'d
|
||||
// straight back to `connecting`; one with no handle yet (parked since download.add
|
||||
// with startMode 'later', or never admitted) goes to `queued` for the next tick's
|
||||
// normal admission.
|
||||
rpc::TaskActionPort::Result user_resume(const std::string& wire_id) override;
|
||||
// "Begin or restart the given tasks" (download.start): same effect as user_resume for
|
||||
// a paused/new task. NOTE: the contract's "a task in 'queued' jumps its queue" priority
|
||||
// bump is not implemented — admission is still plain FIFO via the governor's
|
||||
// created_at rank. Flagged in deferrals.md.
|
||||
rpc::TaskActionPort::Result user_start(const std::string& wire_id) override;
|
||||
// download.cancel == cancel(discard_partial=false); download.remove == cancel(true)
|
||||
// plus the store row / file cleanup (that part is still D3).
|
||||
rpc::TaskActionPort::Result user_cancel(const std::string& wire_id,
|
||||
bool discard_partial) override;
|
||||
|
||||
// queue.stop(pauseRunning=true): pause every task in `queue_id` the governor would
|
||||
// currently call Running, right now rather than waiting for the next tick — the same
|
||||
// immediacy reasoning as the user_* actions above, with PauseReason::QueueStopped
|
||||
// instead of User. Returns the wire ids actually paused.
|
||||
std::vector<std::string> pause_queue(const std::string& queue_id) override;
|
||||
|
||||
bool provide_auth(const std::string& wire_id, const std::string& username,
|
||||
const std::string& password, bool remember) override;
|
||||
|
||||
// rpc::TaskActionPort::apply_settings_reload — a void-returning wrapper around the
|
||||
// already-public reload_config() above (which returns DbResult<void>, consumed by
|
||||
// main.cpp and by sched_scheduler_test; kept as-is rather than changed to match the
|
||||
// port, which has no caller that wants the DbError).
|
||||
void apply_settings_reload() override { (void)reload_config(); }
|
||||
|
||||
void set_global_speed_limit(std::uint64_t bps) override { engine_.set_global_speed_limit(bps); }
|
||||
|
||||
// rpc::TaskActionPort. Builds a vdm::net::ProbeRequest from `params`, runs it on the
|
||||
// engine's probe pool (outside the segment budget, ADR 0011 §5), and converts the
|
||||
// result back to proto terms — including the suggestedCategoryId/-SaveDir guess (a
|
||||
// plain extension match against the categories table; the real rules engine is D3).
|
||||
// `done` is called already marshalled onto the loop thread via post_to_loop, same as
|
||||
// every other engine callback here — the caller never has to know it started on an
|
||||
// engine thread.
|
||||
void probe_now(
|
||||
const velox::proto::DownloadProbeParams& params,
|
||||
std::function<void(velox::proto::HandlerResult<velox::proto::DownloadProbeResult>)>
|
||||
done) override;
|
||||
|
||||
void refresh_url(
|
||||
const std::string& wire_id, const std::string& url,
|
||||
const std::optional<velox::proto::Headers>& headers,
|
||||
const std::optional<std::vector<velox::proto::Cookie>>& cookies,
|
||||
std::function<void(velox::proto::HandlerResult<velox::proto::DownloadRefreshUrlResult>)>
|
||||
done) override;
|
||||
|
||||
// Diagnostics / tests.
|
||||
std::optional<std::string> wire_id_of(vdm::TaskId id) const;
|
||||
@@ -76,9 +170,41 @@ private:
|
||||
void map(const std::string& wire_id, vdm::TaskId engine_id);
|
||||
void unmap_engine(vdm::TaskId engine_id);
|
||||
|
||||
// The one place a task's state row changes and (if a hub is set) event.task.state
|
||||
// publishes. previousState is read from the store's own current row, not passed in —
|
||||
// authoritative regardless of engine/scheduler timing.
|
||||
void transition(const std::string& wire_id, std::string_view to_state,
|
||||
std::optional<std::string> pause_reason,
|
||||
const std::optional<TaskErrorFields>& err);
|
||||
|
||||
// The probe issued for `wire_id` in tick() has resolved: persist sizeBytes /
|
||||
// resumable / validator, then start() with the result as probe_hint. A probe failure
|
||||
// (bad URL, DNS, 404 with no mirrors) moves the task straight to `failed` — it never
|
||||
// reaches start().
|
||||
void on_probe_result(const std::string& wire_id, const vdm::Result<vdm::net::ProbeResult>& pr);
|
||||
|
||||
// on_finished fired: top up the final byte count (set_final_bytes) so a task that
|
||||
// completed before any progress tick ran still reports real numbers. The state
|
||||
// transition itself (complete/failed/cancelled) already happened via on_engine_state,
|
||||
// which on_finished always follows.
|
||||
void on_engine_finished(const std::string& wire_id,
|
||||
const vdm::Result<vdm::task::DownloadOutcome>& outcome);
|
||||
|
||||
// Writes one task's byte counters + segment rows from a Progress snapshot. Shared by
|
||||
// progress_snapshot() (the periodic tick) and on_engine_state's terminal path (a final
|
||||
// snapshot before release/unmap) so a task that finishes between two ticks — the
|
||||
// common case for anything small or fast — still leaves a real segmentDetail behind
|
||||
// instead of the pre-segmentation empty array.
|
||||
void persist_progress(const std::string& wire_id, const vdm::task::Progress& p);
|
||||
|
||||
// Wires on_state -> on_engine_state and on_finished -> on_engine_finished, both
|
||||
// marshalled through post_to_loop. Shared by the one place a task actually starts.
|
||||
vdm::task::DownloadCallbacks make_callbacks(const std::string& wire_id);
|
||||
|
||||
store::Db& db_;
|
||||
EnginePort& engine_;
|
||||
Governor governor_;
|
||||
rpc::EventHub* hub_;
|
||||
Deps deps_;
|
||||
|
||||
std::unordered_map<std::string, vdm::TaskId> to_engine_;
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
#include "store/categories.hpp"
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cstdio>
|
||||
#include <random>
|
||||
#include <string>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
namespace proto = velox::proto;
|
||||
|
||||
namespace {
|
||||
|
||||
proto::Category project_row(Stmt& st) {
|
||||
proto::Category c;
|
||||
c.categoryId = st.column_text(0);
|
||||
c.name = st.column_text(1);
|
||||
c.saveDir = st.column_text(2);
|
||||
auto j = nlohmann::json::parse(st.column_text(3), nullptr, false);
|
||||
if (j.is_array()) {
|
||||
for (const auto& e : j)
|
||||
if (e.is_string()) c.extensions.push_back(e.get<std::string>());
|
||||
}
|
||||
c.builtin = st.column_int(4) != 0;
|
||||
return c;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
DbResult<std::vector<proto::Category>> Categories::list() {
|
||||
auto st = db_.prepare(
|
||||
"SELECT category_id, name, save_dir, extensions, builtin FROM categories "
|
||||
"ORDER BY builtin DESC, name");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
|
||||
std::vector<proto::Category> out;
|
||||
for (;;) {
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) break;
|
||||
out.push_back(project_row(*st));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
DbResult<std::optional<proto::Category>> Categories::get(std::string_view category_id) {
|
||||
auto st = db_.prepare(
|
||||
"SELECT category_id, name, save_dir, extensions, builtin FROM categories "
|
||||
"WHERE category_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, category_id); !b) return std::unexpected(b.error());
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) return std::optional<proto::Category>{};
|
||||
return std::optional<proto::Category>{project_row(*st)};
|
||||
}
|
||||
|
||||
std::string Categories::guess_by_extension(std::string_view filename_or_ext) {
|
||||
std::string ext(filename_or_ext);
|
||||
if (const auto dot = ext.find_last_of('.'); dot != std::string::npos) ext = ext.substr(dot + 1);
|
||||
if (ext.empty()) return "general";
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(),
|
||||
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
|
||||
|
||||
auto cats = list();
|
||||
if (!cats) return "general";
|
||||
for (const auto& c : *cats) {
|
||||
for (const auto& e : c.extensions)
|
||||
if (e == ext) return c.categoryId;
|
||||
}
|
||||
return "general";
|
||||
}
|
||||
|
||||
DbResult<proto::Category> Categories::upsert(proto::Category category) {
|
||||
// A replace keeps the existing row's builtin flag; a create is never builtin. Either
|
||||
// way the payload's own `builtin` is ignored — a client cannot mint or revoke it.
|
||||
bool builtin = false;
|
||||
if (!category.categoryId.empty()) {
|
||||
auto existing = get(category.categoryId);
|
||||
if (!existing) return std::unexpected(existing.error());
|
||||
if (existing->has_value()) builtin = (*existing)->builtin;
|
||||
} else {
|
||||
// Reuse Tasks' id scheme (v4 UUID) would need a cross-module include for one
|
||||
// function; a category id has no wire format requirement beyond "a string", so a
|
||||
// timestamp-free random hex id keeps this module self-contained.
|
||||
std::random_device rd;
|
||||
std::uniform_int_distribution<std::uint64_t> d;
|
||||
char buf[17];
|
||||
std::snprintf(buf, sizeof(buf), "%016llx", static_cast<unsigned long long>(d(rd)));
|
||||
category.categoryId = std::string(buf);
|
||||
}
|
||||
|
||||
nlohmann::json ext = nlohmann::json::array();
|
||||
for (const auto& e : category.extensions) ext.push_back(e);
|
||||
|
||||
auto st = db_.prepare(
|
||||
"INSERT INTO categories(category_id, name, save_dir, extensions, builtin) "
|
||||
"VALUES(?1,?2,?3,?4,?5) "
|
||||
"ON CONFLICT(category_id) DO UPDATE SET "
|
||||
"name=excluded.name, save_dir=excluded.save_dir, extensions=excluded.extensions");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, std::string_view(category.categoryId)); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto b = st->bind(2, std::string_view(category.name)); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(3, std::string_view(category.saveDir)); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto b = st->bind(4, std::string_view(ext.dump())); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(5, static_cast<std::int64_t>(builtin)); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
|
||||
category.builtin = builtin;
|
||||
return category;
|
||||
}
|
||||
|
||||
DbResult<Categories::RemoveResult> Categories::remove(
|
||||
std::string_view category_id, const std::optional<std::string>& reassign_to) {
|
||||
RemoveResult out;
|
||||
const std::string target = reassign_to.value_or("general");
|
||||
|
||||
// Db::transaction only threads a DbResult<void> lambda; `out` is filled in-place and
|
||||
// returned once the transaction (which may still fail and roll back) succeeds.
|
||||
auto txn = db_.transaction([&]() -> DbResult<void> {
|
||||
{
|
||||
// A builtin category is never removed, and — since it was never going to be
|
||||
// removed — its tasks must not be reassigned away from it either.
|
||||
auto chk = db_.prepare("SELECT builtin FROM categories WHERE category_id = ?1");
|
||||
if (!chk) return std::unexpected(chk.error());
|
||||
if (auto b = chk->bind(1, category_id); !b) return std::unexpected(b.error());
|
||||
auto row = chk->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) return {}; // no such category: removed stays false
|
||||
if (chk->column_int(0) != 0) return {}; // builtin: removed stays false
|
||||
}
|
||||
{
|
||||
auto sel = db_.prepare("SELECT task_id FROM tasks WHERE category_id = ?1");
|
||||
if (!sel) return std::unexpected(sel.error());
|
||||
if (auto b = sel->bind(1, category_id); !b) return std::unexpected(b.error());
|
||||
for (;;) {
|
||||
auto row = sel->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) break;
|
||||
out.reassigned_task_ids.push_back(sel->column_text(0));
|
||||
}
|
||||
}
|
||||
if (!out.reassigned_task_ids.empty()) {
|
||||
auto upd = db_.prepare("UPDATE tasks SET category_id = ?2 WHERE category_id = ?1");
|
||||
if (!upd) return std::unexpected(upd.error());
|
||||
if (auto b = upd->bind(1, category_id); !b) return std::unexpected(b.error());
|
||||
if (auto b = upd->bind(2, std::string_view(target)); !b) return std::unexpected(b.error());
|
||||
if (auto r = upd->step(); !r) return std::unexpected(r.error());
|
||||
}
|
||||
auto del = db_.prepare("DELETE FROM categories WHERE category_id = ?1 AND builtin = 0");
|
||||
if (!del) return std::unexpected(del.error());
|
||||
if (auto b = del->bind(1, category_id); !b) return std::unexpected(b.error());
|
||||
if (auto r = del->step(); !r) return std::unexpected(r.error());
|
||||
out.removed = sqlite3_changes(db_.raw()) > 0;
|
||||
return {};
|
||||
});
|
||||
if (!txn) return std::unexpected(txn.error());
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,56 @@
|
||||
#pragma once
|
||||
|
||||
// Read access to the `categories` table, projected onto proto::Category. Owned here
|
||||
// rather than duplicated per handler since category.list and download.add (rule
|
||||
// matching, later) both need it.
|
||||
//
|
||||
// The table has no columns for Category.mimeTypes / .sortOrder (0001_initial.sql predates
|
||||
// those fields); upsert() accepts them but they are not persisted — round-tripped as unset
|
||||
// on the next list()/get(). Noted in daemon/docs/deferrals.md.
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "store/sqlite.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
class Categories {
|
||||
public:
|
||||
explicit Categories(Db& db) : db_(db) {}
|
||||
|
||||
DbResult<std::vector<velox::proto::Category>> list();
|
||||
DbResult<std::optional<velox::proto::Category>> get(std::string_view category_id);
|
||||
|
||||
// Extension match against categories.extensions — not the real rules engine (no
|
||||
// host/mime/size clauses), just enough that a capture or a File Info preselect isn't
|
||||
// always "general". `filename_or_ext` may be a whole filename ("movie.mp4") or a bare
|
||||
// extension ("mp4", no leading dot); matched case-insensitively. "general" (this
|
||||
// project's always-present default category) on no match, an empty/dotless filename,
|
||||
// or a store error — this never fails outward, it just falls back.
|
||||
std::string guess_by_extension(std::string_view filename_or_ext);
|
||||
|
||||
// "Omit categoryId to create; supply it to replace" (category.upsert's own words) —
|
||||
// the caller (dispatcher) decides create vs replace by whether `category.categoryId`
|
||||
// is empty and generates the id; this just writes the row. `builtin` is never taken
|
||||
// from the payload: preserved from the existing row on a replace, always false on a
|
||||
// create (a client can never mint a builtin category).
|
||||
DbResult<velox::proto::Category> upsert(velox::proto::Category category);
|
||||
|
||||
// False for "no such category". A builtin category is never removed — the caller
|
||||
// checks that (category.remove -> -32602) before calling this, since that check needs
|
||||
// ErrorCode, which this module (like the rest of store/) does not depend on.
|
||||
struct RemoveResult {
|
||||
bool removed = false;
|
||||
std::vector<std::string> reassigned_task_ids;
|
||||
};
|
||||
DbResult<RemoveResult> remove(std::string_view category_id,
|
||||
const std::optional<std::string>& reassign_to);
|
||||
|
||||
private:
|
||||
Db& db_;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,30 @@
|
||||
-- Migration 0002 — segments gets a speed_bps column and a corrected state CHECK;
|
||||
-- tasks gets a speed_bps column too.
|
||||
--
|
||||
-- 0001's segments.state CHECK omitted 'pending' (vdm::segment::SegState::idle's wire
|
||||
-- spelling — "range assigned, no worker connected yet"), so a segment snapshot taken
|
||||
-- before its first worker connects could never be written. SQLite cannot ALTER a CHECK
|
||||
-- constraint in place, so this rebuilds the table (standard SQLite pattern: create the
|
||||
-- new shape, copy, drop, rename). speed_bps on both tables backs TaskSummary.speedBps /
|
||||
-- Segment.speedBps on the wire; absent from 0001 because progress writing wasn't wired
|
||||
-- yet — a task's engine-reported aggregate speed had nowhere to persist between polls.
|
||||
|
||||
ALTER TABLE tasks ADD COLUMN speed_bps INTEGER NOT NULL DEFAULT 0;
|
||||
|
||||
CREATE TABLE segments_new (
|
||||
task_id TEXT NOT NULL REFERENCES tasks(task_id) ON DELETE CASCADE,
|
||||
idx INTEGER NOT NULL,
|
||||
start_byte INTEGER NOT NULL,
|
||||
end_byte INTEGER NOT NULL,
|
||||
completed_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
speed_bps INTEGER NOT NULL DEFAULT 0,
|
||||
state TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (state IN ('pending','connecting','downloading','stalled','complete','failed')),
|
||||
PRIMARY KEY (task_id, idx)
|
||||
) STRICT, WITHOUT ROWID;
|
||||
|
||||
INSERT INTO segments_new (task_id, idx, start_byte, end_byte, completed_bytes, state)
|
||||
SELECT task_id, idx, start_byte, end_byte, completed_bytes, state FROM segments;
|
||||
|
||||
DROP TABLE segments;
|
||||
ALTER TABLE segments_new RENAME TO segments;
|
||||
@@ -0,0 +1,91 @@
|
||||
-- Migration 0003 — tasks.start_mode is rebuilt to the contract's StartMode values.
|
||||
--
|
||||
-- 0001's CHECK read `start_mode IN ('auto','now','queue','manual')`. That is not
|
||||
-- StartMode.schema.json's enum at all: the contract is ['now','later','queue']. The
|
||||
-- practical effect: `download.add` with `startMode: "later"` — a real, documented value
|
||||
-- (the File Info dialog's Download Later button) — hit the CHECK constraint on insert
|
||||
-- and surfaced as an unhandled -32603, every time. 'auto' and 'manual' were never
|
||||
-- contract values; they were this table's own invention and nothing on the wire ever
|
||||
-- sends them.
|
||||
--
|
||||
-- SQLite cannot ALTER a CHECK constraint in place, so this rebuilds the table (same
|
||||
-- pattern as 0002: create the new shape, copy with the value mapped, drop, rename).
|
||||
-- Existing rows are remapped by what they actually meant: 'auto' was "eligible for the
|
||||
-- scheduler the moment it's added", i.e. 'now'; 'manual' was "parked, wait for the user",
|
||||
-- which is what 'later' means on the wire (StartMode's own description: "lands the task
|
||||
-- in paused"). Any row already spelled 'now' or 'queue' passes through unchanged.
|
||||
--
|
||||
-- This does NOT touch `state` or `pause_reason` — a row that was start_mode='manual' and
|
||||
-- (per the dispatcher's now-dead branch) state='new' keeps state='new'; the daemon-side
|
||||
-- fix to actually land a 'later' task in 'paused' going forward lives in dispatcher.cpp,
|
||||
-- not in this migration. Historical rows are not replayed through the scheduler.
|
||||
|
||||
CREATE TABLE tasks_new (
|
||||
task_id TEXT PRIMARY KEY,
|
||||
url TEXT NOT NULL,
|
||||
effective_url TEXT,
|
||||
filename TEXT NOT NULL DEFAULT '',
|
||||
save_dir TEXT NOT NULL,
|
||||
category_id TEXT REFERENCES categories(category_id) ON DELETE SET NULL,
|
||||
queue_id TEXT REFERENCES queues(queue_id) ON DELETE SET NULL,
|
||||
queue_position INTEGER,
|
||||
|
||||
state TEXT NOT NULL DEFAULT 'new'
|
||||
CHECK (state IN ('new','probing','queued','connecting','downloading','paused',
|
||||
'retry_wait','assembling','verifying','complete','failed','cancelled')),
|
||||
pause_reason TEXT CHECK (pause_reason IN
|
||||
('user','schedule','queue_stopped','admission_reconcile','auto')),
|
||||
|
||||
size_bytes INTEGER,
|
||||
downloaded_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
resumable INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
req_segments INTEGER,
|
||||
eff_segments INTEGER NOT NULL DEFAULT 0,
|
||||
req_buffer_bytes INTEGER,
|
||||
eff_buffer_bytes INTEGER,
|
||||
|
||||
-- The contract's StartMode (StartMode.schema.json): 'now' | 'later' | 'queue'.
|
||||
start_mode TEXT NOT NULL DEFAULT 'now'
|
||||
CHECK (start_mode IN ('now','later','queue')),
|
||||
description TEXT,
|
||||
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
content_type TEXT,
|
||||
|
||||
checksum_algo TEXT CHECK (checksum_algo IN ('md5','sha1','sha256','sha512')),
|
||||
checksum_value TEXT,
|
||||
|
||||
error_code TEXT,
|
||||
error_message TEXT,
|
||||
error_http_status INTEGER,
|
||||
error_retryable INTEGER,
|
||||
error_attempt INTEGER,
|
||||
error_next_retry_at TEXT,
|
||||
|
||||
speed_bps INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
created_at TEXT NOT NULL,
|
||||
last_try_at TEXT,
|
||||
completed_at TEXT
|
||||
) STRICT;
|
||||
|
||||
INSERT INTO tasks_new
|
||||
SELECT task_id, url, effective_url, filename, save_dir, category_id, queue_id,
|
||||
queue_position, state, pause_reason, size_bytes, downloaded_bytes, resumable,
|
||||
req_segments, eff_segments, req_buffer_bytes, eff_buffer_bytes,
|
||||
CASE start_mode WHEN 'auto' THEN 'now' WHEN 'manual' THEN 'later' ELSE start_mode END,
|
||||
description, etag, last_modified, content_type, checksum_algo, checksum_value,
|
||||
error_code, error_message, error_http_status, error_retryable, error_attempt,
|
||||
error_next_retry_at, speed_bps, created_at, last_try_at, completed_at
|
||||
FROM tasks;
|
||||
|
||||
DROP TABLE tasks;
|
||||
ALTER TABLE tasks_new RENAME TO tasks;
|
||||
|
||||
CREATE INDEX idx_tasks_state ON tasks(state);
|
||||
CREATE INDEX idx_tasks_category ON tasks(category_id);
|
||||
CREATE INDEX idx_tasks_queue_order ON tasks(queue_id, queue_position);
|
||||
CREATE INDEX idx_tasks_created ON tasks(created_at);
|
||||
CREATE INDEX idx_tasks_completed ON tasks(completed_at);
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Migration 0004 — rules gets a name column.
|
||||
--
|
||||
-- 0001's rules table had no column for Rule.name (Rule.schema.json's own optional,
|
||||
-- maxLength-64 label field) — store/rules.cpp discovered this the hard way building
|
||||
-- rules.list/rules.upsert: every rules.list call failed outright ("no such column:
|
||||
-- name") because the SELECT it needs to project onto proto::Rule names a column that was
|
||||
-- never there. A plain ALTER TABLE ADD COLUMN suffices here (no CHECK constraint to
|
||||
-- rebuild around, unlike 0002/0003).
|
||||
|
||||
ALTER TABLE rules ADD COLUMN name TEXT;
|
||||
@@ -0,0 +1,183 @@
|
||||
#include "store/queues.hpp"
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <random>
|
||||
#include <string>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
namespace proto = velox::proto;
|
||||
|
||||
namespace {
|
||||
|
||||
// One queue row (columns queue_id, name, state, max_concurrent, schedule, on_complete, in
|
||||
// that order) plus its member taskIds, read off the row a caller has already step()'d to.
|
||||
DbResult<proto::Queue> project_row(Db& db, Stmt& st) {
|
||||
proto::Queue q;
|
||||
q.queueId = st.column_text(0);
|
||||
q.name = st.column_text(1);
|
||||
if (auto s = proto::parse_QueueState(st.column_text(2))) q.state = *s;
|
||||
q.maxConcurrent = st.column_int(3);
|
||||
if (!st.column_is_null(4)) {
|
||||
auto j = nlohmann::json::parse(st.column_text(4), nullptr, false);
|
||||
if (auto sched = proto::parse<proto::Schedule>(j, "schedule")) q.schedule = *sched;
|
||||
}
|
||||
if (auto oc = proto::parse_QueueOnComplete(st.column_text(5))) q.onComplete = *oc;
|
||||
|
||||
auto ts = db.prepare("SELECT task_id FROM tasks WHERE queue_id = ?1 ORDER BY queue_position");
|
||||
if (!ts) return std::unexpected(ts.error());
|
||||
if (auto b = ts->bind(1, std::string_view(q.queueId)); !b) return std::unexpected(b.error());
|
||||
std::vector<std::string> ids;
|
||||
for (;;) {
|
||||
auto r = ts->step();
|
||||
if (!r) return std::unexpected(r.error());
|
||||
if (!*r) break;
|
||||
ids.push_back(ts->column_text(0));
|
||||
}
|
||||
q.taskIds = std::move(ids);
|
||||
return q;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
DbResult<std::vector<proto::Queue>> Queues::list() {
|
||||
auto st = db_.prepare(
|
||||
"SELECT queue_id, name, state, max_concurrent, schedule, on_complete FROM queues "
|
||||
"ORDER BY name");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
|
||||
std::vector<proto::Queue> out;
|
||||
for (;;) {
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) break;
|
||||
auto q = project_row(db_, *st);
|
||||
if (!q) return std::unexpected(q.error());
|
||||
out.push_back(std::move(*q));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
DbResult<std::optional<proto::Queue>> Queues::get(std::string_view queue_id) {
|
||||
auto st = db_.prepare(
|
||||
"SELECT queue_id, name, state, max_concurrent, schedule, on_complete FROM queues "
|
||||
"WHERE queue_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, queue_id); !b) return std::unexpected(b.error());
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) return std::optional<proto::Queue>{};
|
||||
auto q = project_row(db_, *st);
|
||||
if (!q) return std::unexpected(q.error());
|
||||
return std::optional<proto::Queue>{std::move(*q)};
|
||||
}
|
||||
|
||||
DbResult<bool> Queues::set_state(std::string_view queue_id, std::string_view state) {
|
||||
auto st = db_.prepare("UPDATE queues SET state = ?2 WHERE queue_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, queue_id); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(2, state); !b) return std::unexpected(b.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Queues::set_schedule(std::string_view queue_id,
|
||||
const std::optional<proto::Schedule>& schedule) {
|
||||
auto st = db_.prepare("UPDATE queues SET schedule = ?2 WHERE queue_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, queue_id); !b) return std::unexpected(b.error());
|
||||
if (auto r = schedule ? st->bind(2, std::string_view(nlohmann::json(*schedule).dump()))
|
||||
: st->bind_null(2);
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Queues::reorder(std::string_view queue_id, const std::vector<std::string>& task_ids) {
|
||||
std::vector<std::string> current;
|
||||
{
|
||||
auto st = db_.prepare(
|
||||
"SELECT task_id FROM tasks WHERE queue_id = ?1 ORDER BY queue_position");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, queue_id); !b) return std::unexpected(b.error());
|
||||
for (;;) {
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) break;
|
||||
current.push_back(st->column_text(0));
|
||||
}
|
||||
}
|
||||
|
||||
// Exact permutation: same size, same members, order aside.
|
||||
std::vector<std::string> a = current, b = task_ids;
|
||||
std::sort(a.begin(), a.end());
|
||||
std::sort(b.begin(), b.end());
|
||||
if (a != b) return false;
|
||||
|
||||
auto txn = db_.transaction([&]() -> DbResult<void> {
|
||||
for (std::size_t i = 0; i < task_ids.size(); ++i) {
|
||||
auto st = db_.prepare("UPDATE tasks SET queue_position = ?2 WHERE task_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto bd = st->bind(1, std::string_view(task_ids[i])); !bd)
|
||||
return std::unexpected(bd.error());
|
||||
if (auto bd = st->bind(2, static_cast<std::int64_t>(i)); !bd)
|
||||
return std::unexpected(bd.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
}
|
||||
return {};
|
||||
});
|
||||
if (!txn) return std::unexpected(txn.error());
|
||||
return true;
|
||||
}
|
||||
|
||||
DbResult<proto::Queue> Queues::upsert(proto::Queue queue) {
|
||||
if (queue.queueId.empty()) {
|
||||
std::random_device rd;
|
||||
std::uniform_int_distribution<std::uint64_t> d;
|
||||
char buf[17];
|
||||
std::snprintf(buf, sizeof(buf), "%016llx", static_cast<unsigned long long>(d(rd)));
|
||||
queue.queueId = std::string(buf);
|
||||
}
|
||||
|
||||
// A create defaults to 'stopped' (never auto-runs a brand-new queue); a replace keeps
|
||||
// whatever run state the queue is already in — queue.upsert edits the config, not the
|
||||
// run state (that's queue.start/stop).
|
||||
std::string state = "stopped";
|
||||
if (auto existing = get(queue.queueId); existing && existing->has_value())
|
||||
state = std::string(proto::to_string((*existing)->state));
|
||||
|
||||
const std::string schedule_json =
|
||||
queue.schedule ? nlohmann::json(*queue.schedule).dump() : std::string();
|
||||
const std::string on_complete =
|
||||
std::string(proto::to_string(queue.onComplete.value_or(proto::QueueOnComplete::Nothing)));
|
||||
|
||||
auto st = db_.prepare(
|
||||
"INSERT INTO queues(queue_id, name, state, max_concurrent, schedule, on_complete) "
|
||||
"VALUES(?1,?2,?3,?4,?5,?6) "
|
||||
"ON CONFLICT(queue_id) DO UPDATE SET "
|
||||
"name=excluded.name, max_concurrent=excluded.max_concurrent, "
|
||||
"schedule=excluded.schedule, on_complete=excluded.on_complete");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, std::string_view(queue.queueId)); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(2, std::string_view(queue.name)); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(3, std::string_view(state)); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(4, queue.maxConcurrent); !b) return std::unexpected(b.error());
|
||||
if (auto r = queue.schedule ? st->bind(5, std::string_view(schedule_json)) : st->bind_null(5); !r)
|
||||
return std::unexpected(r.error());
|
||||
if (auto b = st->bind(6, std::string_view(on_complete)); !b) return std::unexpected(b.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
|
||||
auto stored = get(queue.queueId);
|
||||
if (!stored) return std::unexpected(stored.error());
|
||||
if (!stored->has_value())
|
||||
return std::unexpected(DbError{0, "queue.upsert: row vanished after insert"});
|
||||
return **stored;
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,52 @@
|
||||
#pragma once
|
||||
|
||||
// Read access to the `queues` table, projected onto proto::Queue. taskIds is derived from
|
||||
// `tasks` (queue_id = this queue, ordered by queue_position), not stored on the queue row
|
||||
// — membership changes through download.update / queue.reorder, per Queue's own schema
|
||||
// note that a queue.upsert payload's taskIds is ignored.
|
||||
|
||||
#include <optional>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "store/sqlite.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
class Queues {
|
||||
public:
|
||||
explicit Queues(Db& db) : db_(db) {}
|
||||
|
||||
DbResult<std::vector<velox::proto::Queue>> list();
|
||||
|
||||
// nullopt (not an error) if no queue has this id.
|
||||
DbResult<std::optional<velox::proto::Queue>> get(std::string_view queue_id);
|
||||
|
||||
// 'running' or 'stopped' (Queue.schema.json / the state column's CHECK). false if the
|
||||
// id doesn't exist.
|
||||
DbResult<bool> set_state(std::string_view queue_id, std::string_view state);
|
||||
|
||||
// schedule.set: nullopt clears it (NULL = manual control, per the schema's own
|
||||
// words). false if the queue doesn't exist.
|
||||
DbResult<bool> set_schedule(std::string_view queue_id,
|
||||
const std::optional<velox::proto::Schedule>& schedule);
|
||||
|
||||
// queue.reorder: `task_ids` must be an exact permutation of the queue's current
|
||||
// membership (the schema's own words — "anything else is -32602 rather than a
|
||||
// partial reorder, so a stale drag from an out-of-date view cannot quietly reshuffle
|
||||
// the queue"). false (no write at all) if it isn't; true and queue_position rewritten
|
||||
// to match `task_ids`'s order if it is.
|
||||
DbResult<bool> reorder(std::string_view queue_id, const std::vector<std::string>& task_ids);
|
||||
|
||||
// "Omit queueId to create" (queue.upsert's own words) — an empty id generates one.
|
||||
// taskIds is ignored (membership changes only through download.update / queue.reorder,
|
||||
// per the schema's own note); a create defaults to 'stopped', a replace keeps the
|
||||
// queue's current run state (queue.upsert edits config, not run state).
|
||||
DbResult<velox::proto::Queue> upsert(velox::proto::Queue queue);
|
||||
|
||||
private:
|
||||
Db& db_;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,96 @@
|
||||
#include "store/rules.hpp"
|
||||
|
||||
#include <cstdio>
|
||||
#include <random>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
namespace proto = velox::proto;
|
||||
|
||||
namespace {
|
||||
|
||||
proto::Rule project_row(Stmt& st) {
|
||||
proto::Rule r;
|
||||
r.ruleId = st.column_text(0);
|
||||
if (!st.column_is_null(1)) r.name = st.column_text(1);
|
||||
r.enabled = st.column_int(2) != 0;
|
||||
r.priority = st.column_int(3);
|
||||
if (auto j = nlohmann::json::parse(st.column_text(4), nullptr, false); !j.is_discarded()) {
|
||||
if (auto m = proto::parse<proto::RuleMatch>(j, "match")) r.match = *m;
|
||||
}
|
||||
if (auto j = nlohmann::json::parse(st.column_text(5), nullptr, false); !j.is_discarded()) {
|
||||
if (auto a = proto::parse<proto::RuleAction>(j, "action")) r.action = *a;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
std::string new_rule_id() {
|
||||
std::random_device rd;
|
||||
std::uniform_int_distribution<std::uint64_t> d;
|
||||
char buf[17];
|
||||
std::snprintf(buf, sizeof(buf), "%016llx", static_cast<unsigned long long>(d(rd)));
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
DbResult<std::vector<proto::Rule>> Rules::list() {
|
||||
auto st = db_.prepare(
|
||||
"SELECT rule_id, name, enabled, priority, match, action FROM rules "
|
||||
"ORDER BY priority, rule_id");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
|
||||
std::vector<proto::Rule> out;
|
||||
for (;;) {
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) break;
|
||||
out.push_back(project_row(*st));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
DbResult<std::vector<proto::Rule>> Rules::apply(std::vector<proto::Rule> upsert,
|
||||
const std::vector<std::string>& remove) {
|
||||
auto txn = db_.transaction([&]() -> DbResult<void> {
|
||||
for (const auto& id : remove) {
|
||||
auto del = db_.prepare("DELETE FROM rules WHERE rule_id = ?1");
|
||||
if (!del) return std::unexpected(del.error());
|
||||
if (auto b = del->bind(1, std::string_view(id)); !b) return std::unexpected(b.error());
|
||||
if (auto r = del->step(); !r) return std::unexpected(r.error());
|
||||
}
|
||||
for (auto& rule : upsert) {
|
||||
if (rule.ruleId.empty()) rule.ruleId = new_rule_id();
|
||||
|
||||
nlohmann::json match_json = rule.match;
|
||||
nlohmann::json action_json = rule.action;
|
||||
|
||||
auto st = db_.prepare(
|
||||
"INSERT INTO rules(rule_id, name, enabled, priority, match, action) "
|
||||
"VALUES(?1,?2,?3,?4,?5,?6) "
|
||||
"ON CONFLICT(rule_id) DO UPDATE SET "
|
||||
"name=excluded.name, enabled=excluded.enabled, priority=excluded.priority, "
|
||||
"match=excluded.match, action=excluded.action");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, std::string_view(rule.ruleId)); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto r = rule.name ? st->bind(2, std::string_view(*rule.name)) : st->bind_null(2); !r)
|
||||
return std::unexpected(r.error());
|
||||
if (auto b = st->bind(3, static_cast<std::int64_t>(rule.enabled)); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto b = st->bind(4, rule.priority); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(5, std::string_view(match_json.dump())); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto b = st->bind(6, std::string_view(action_json.dump())); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
}
|
||||
return {};
|
||||
});
|
||||
if (!txn) return std::unexpected(txn.error());
|
||||
return list();
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,37 @@
|
||||
#pragma once
|
||||
|
||||
// Read/write access to the `rules` table (rules.list / rules.upsert / capture.offer's own
|
||||
// read path). match/action are stored as their generated-JSON text (velox::proto::RuleMatch
|
||||
// / RuleAction), so no bespoke schema lives here beyond the table's own columns.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "store/sqlite.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
class Rules {
|
||||
public:
|
||||
explicit Rules(Db& db) : db_(db) {}
|
||||
|
||||
// Enabled and disabled rules alike, in priority order (ties broken by rule_id) —
|
||||
// rules.list's own contract ("the rules engine's table, in priority order"); capture
|
||||
// offer's own caller filters to enabled ones itself, same as vdm::rules::match_rules
|
||||
// already does internally.
|
||||
DbResult<std::vector<velox::proto::Rule>> list();
|
||||
|
||||
// rules.upsert: "upsert carries the rules to store and remove the ruleIds to drop;
|
||||
// applying both at once means a reprioritisation never leaves the table in a
|
||||
// half-valid state" (the schema's own words) — one transaction. An empty ruleId in
|
||||
// `upsert` generates one (create); a non-empty one replaces. Returns the full table
|
||||
// after the write, in priority order.
|
||||
DbResult<std::vector<velox::proto::Rule>> apply(std::vector<velox::proto::Rule> upsert,
|
||||
const std::vector<std::string>& remove);
|
||||
|
||||
private:
|
||||
Db& db_;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,61 @@
|
||||
#include "store/segments.hpp"
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
namespace proto = velox::proto;
|
||||
|
||||
DbResult<void> Segments::replace_all(std::string_view task_id,
|
||||
const std::vector<SegmentSnapshot>& segs) {
|
||||
return db_.transaction([&]() -> DbResult<void> {
|
||||
{
|
||||
auto del = db_.prepare("DELETE FROM segments WHERE task_id = ?1");
|
||||
if (!del) return std::unexpected(del.error());
|
||||
if (auto r = del->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = del->step(); !r) return std::unexpected(r.error());
|
||||
}
|
||||
for (const auto& s : segs) {
|
||||
auto ins = db_.prepare(
|
||||
"INSERT INTO segments(task_id, idx, start_byte, end_byte, completed_bytes, "
|
||||
"speed_bps, state) VALUES(?1,?2,?3,?4,?5,?6,?7)");
|
||||
if (!ins) return std::unexpected(ins.error());
|
||||
if (auto r = ins->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = ins->bind(2, static_cast<std::int64_t>(s.index)); !r)
|
||||
return std::unexpected(r.error());
|
||||
if (auto r = ins->bind(3, s.start_byte); !r) return std::unexpected(r.error());
|
||||
if (auto r = ins->bind(4, s.end_byte); !r) return std::unexpected(r.error());
|
||||
if (auto r = ins->bind(5, s.completed_bytes); !r) return std::unexpected(r.error());
|
||||
if (auto r = ins->bind(6, s.speed_bps); !r) return std::unexpected(r.error());
|
||||
if (auto r = ins->bind(7, std::string_view(s.state)); !r)
|
||||
return std::unexpected(r.error());
|
||||
if (auto r = ins->step(); !r) return std::unexpected(r.error());
|
||||
}
|
||||
return {};
|
||||
});
|
||||
}
|
||||
|
||||
DbResult<std::vector<proto::Segment>> Segments::list(std::string_view task_id) {
|
||||
auto st = db_.prepare(
|
||||
"SELECT idx, start_byte, end_byte, completed_bytes, speed_bps, state "
|
||||
"FROM segments WHERE task_id = ?1 ORDER BY idx");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto r = st->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
|
||||
std::vector<proto::Segment> out;
|
||||
for (;;) {
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) break;
|
||||
|
||||
proto::Segment seg;
|
||||
seg.index = st->column_int(0);
|
||||
seg.startByte = st->column_int(1);
|
||||
seg.endByte = st->column_int(2);
|
||||
seg.downloadedBytes = st->column_int(3);
|
||||
seg.speedBps = st->column_int(4);
|
||||
if (auto s = proto::parse_SegmentState(st->column_text(5))) seg.state = *s;
|
||||
out.push_back(seg);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,45 @@
|
||||
#pragma once
|
||||
|
||||
// Read/write access to the `segments` table — the per-connection detail behind
|
||||
// TaskDetail.segmentDetail (download.get) and the segment-bars widget. Written from an
|
||||
// engine progress tick (Scheduler::progress_snapshot); read back on download.get.
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "store/sqlite.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
// One segment's live counters, as read off vdm::task::SegmentProgress. `state` is a wire
|
||||
// SegmentState spelling ("pending"/"connecting"/.../"failed") — the caller maps the
|
||||
// engine's own enum, this module stays generic like the others.
|
||||
struct SegmentSnapshot {
|
||||
std::uint32_t index;
|
||||
std::int64_t start_byte;
|
||||
std::int64_t end_byte; // inclusive (ADR 0010)
|
||||
std::int64_t completed_bytes;
|
||||
std::int64_t speed_bps;
|
||||
std::string state;
|
||||
};
|
||||
|
||||
class Segments {
|
||||
public:
|
||||
explicit Segments(Db& db) : db_(db) {}
|
||||
|
||||
// Replaces every row for `task_id` with `segs` in one transaction. Simpler than a
|
||||
// per-index upsert and cheap at <=32 rows, <=4 Hz.
|
||||
DbResult<void> replace_all(std::string_view task_id, const std::vector<SegmentSnapshot>& segs);
|
||||
|
||||
// In index order. Empty if the task has never been segmented (TaskDetail's own
|
||||
// description permits this).
|
||||
DbResult<std::vector<velox::proto::Segment>> list(std::string_view task_id);
|
||||
|
||||
private:
|
||||
Db& db_;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -9,24 +9,62 @@ namespace velox::daemon::store {
|
||||
|
||||
namespace {
|
||||
|
||||
// Built-in defaults, mirroring Settings.schema.json / ADR 0012. Only the keys the daemon
|
||||
// currently reads or is likely to need before the full settings.get handler lands; the
|
||||
// rest resolve through the schema's own defaults at that layer.
|
||||
constexpr std::array<std::pair<std::string_view, std::string_view>, 14> kDefaults{{
|
||||
// Built-in defaults, one per SettingKey (Settings.schema.json / SettingKey.schema.json).
|
||||
// The schema itself carries no "default" keyword anywhere — these are what settings.get
|
||||
// falls back to for a key with no stored row, chosen per ADR 0012 where it speaks (the
|
||||
// connection.* buffer/segment keys) and otherwise the conservative, least-surprising
|
||||
// value for that key's own description. capture.monitoredExtensions defaults to the union
|
||||
// of every builtin category's extensions (0001_initial.sql's seed) rather than an
|
||||
// arbitrary list of its own, so the two stay in sync without a second place to edit.
|
||||
constexpr std::array<std::pair<std::string_view, std::string_view>, 43> kDefaults{{
|
||||
{"general.launchOnLogin", "false"},
|
||||
{"general.minimizeToTray", "false"},
|
||||
{"general.showDropTarget", "true"},
|
||||
{"general.confirmOnExit", "true"},
|
||||
{"general.language", "\"system\""},
|
||||
{"general.checkForUpdates", "true"},
|
||||
{"capture.enabled", "true"},
|
||||
{"capture.monitoredExtensions",
|
||||
"[\"exe\",\"msi\",\"deb\",\"rpm\",\"dmg\",\"appimage\",\"iso\",\"zip\",\"tar\",\"gz\","
|
||||
"\"xz\",\"7z\",\"mp4\",\"mkv\",\"webm\",\"avi\",\"mov\",\"flv\",\"m4v\",\"ts\",\"mp3\","
|
||||
"\"flac\",\"aac\",\"ogg\",\"opus\",\"wav\",\"m4a\",\"pdf\",\"doc\",\"docx\",\"xls\","
|
||||
"\"xlsx\",\"ppt\",\"pptx\",\"odt\",\"epub\",\"jpg\",\"jpeg\",\"png\",\"gif\",\"webp\","
|
||||
"\"svg\",\"bmp\",\"tiff\"]"},
|
||||
{"capture.monitoredMimeTypes", "[]"},
|
||||
{"capture.minSizeBytes", "0"},
|
||||
{"capture.excludedHosts", "[]"},
|
||||
{"capture.bypassModifier", "\"shift\""},
|
||||
{"capture.autoStartTypes", "[]"},
|
||||
{"saveTo.defaultDir", "\"~/Downloads\""},
|
||||
{"saveTo.tempDir", "\"\""},
|
||||
{"saveTo.allowedRoots", "[\"~/Downloads\"]"},
|
||||
{"saveTo.fileExistsPolicy", "\"ask\""},
|
||||
{"saveTo.createSubfolderPerSite", "false"},
|
||||
{"connection.preset", "\"auto\""},
|
||||
{"connection.maxSegmentsPerDownload", "8"},
|
||||
{"connection.bufferBytes", "1048576"},
|
||||
{"connection.maxConcurrentDownloads", "5"},
|
||||
{"connection.maxActiveSegments", "32"},
|
||||
{"connection.maxTotalBufferBytes", "134217728"},
|
||||
{"connection.maxActiveSegments", "32"},
|
||||
{"connection.maxConcurrentDownloads", "5"},
|
||||
{"connection.timeoutSec", "30"},
|
||||
{"connection.maxRetries", "10"},
|
||||
{"connection.retryBackoffSec", "5"},
|
||||
{"saveTo.defaultDir", "\"~/Downloads\""},
|
||||
{"saveTo.allowedRoots", "[\"~/Downloads\"]"},
|
||||
{"saveTo.createSubfolderPerSite", "false"},
|
||||
{"downloads.speedLimitBps", "0"},
|
||||
{"downloads.speedLimitEnabled", "false"},
|
||||
{"downloads.virusScanCommand", "\"\""},
|
||||
{"downloads.postDownloadCommand", "\"\""},
|
||||
{"downloads.duplicatePolicy", "\"ask\""},
|
||||
{"downloads.verifyChecksums", "true"},
|
||||
{"proxy.mode", "\"system\""},
|
||||
{"proxy.host", "\"\""},
|
||||
{"proxy.port", "1"},
|
||||
{"proxy.username", "\"\""},
|
||||
{"proxy.bypassHosts", "[]"},
|
||||
{"proxy.pacUrl", "\"\""},
|
||||
{"sounds.enabled", "true"},
|
||||
{"sounds.onComplete", "\"\""},
|
||||
{"sounds.onQueueComplete", "\"\""},
|
||||
{"sounds.onError", "\"\""},
|
||||
}};
|
||||
|
||||
} // namespace
|
||||
@@ -86,6 +124,19 @@ std::int64_t Settings::get_int(std::string_view key) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool Settings::get_bool(std::string_view key) {
|
||||
auto raw = get_raw(key);
|
||||
if (raw && *raw) {
|
||||
auto j = nlohmann::json::parse(**raw, nullptr, false);
|
||||
if (j.is_boolean()) return j.get<bool>();
|
||||
}
|
||||
if (auto d = default_for(key)) {
|
||||
auto j = nlohmann::json::parse(*d, nullptr, false);
|
||||
if (j.is_boolean()) return j.get<bool>();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string Settings::get_string(std::string_view key) {
|
||||
auto raw = get_raw(key);
|
||||
if (raw && *raw) {
|
||||
|
||||
@@ -35,6 +35,7 @@ public:
|
||||
// Typed convenience over get_raw + the defaults. A malformed stored value falls back
|
||||
// to the default rather than throwing.
|
||||
std::int64_t get_int(std::string_view key);
|
||||
bool get_bool(std::string_view key);
|
||||
std::string get_string(std::string_view key);
|
||||
std::vector<std::string> get_string_array(std::string_view key);
|
||||
|
||||
|
||||
+152
-6
@@ -19,7 +19,7 @@ constexpr const char* kCols =
|
||||
"size_bytes, downloaded_bytes, resumable, "
|
||||
"req_segments, eff_segments, req_buffer_bytes, eff_buffer_bytes, queue_position, "
|
||||
"error_code, error_message, error_http_status, error_retryable, error_attempt, "
|
||||
"error_next_retry_at";
|
||||
"error_next_retry_at, speed_bps";
|
||||
|
||||
DbResult<void> bind_opt(Stmt& s, int i, const std::optional<std::string>& v) {
|
||||
return v ? s.bind(i, std::string_view(*v)) : s.bind_null(i);
|
||||
@@ -72,6 +72,7 @@ TaskRow read_row(Stmt& s) {
|
||||
if (!s.column_is_null(30)) r.error_retryable = s.column_int(30) != 0;
|
||||
r.error_attempt = col_opt_int(s, 31);
|
||||
r.error_next_retry_at = col_opt_text(s, 32);
|
||||
r.speed_bps = s.column_int(33);
|
||||
return r;
|
||||
}
|
||||
|
||||
@@ -103,9 +104,9 @@ DbResult<void> Tasks::insert(const TaskRow& r) {
|
||||
"checksum_algo, checksum_value, size_bytes, downloaded_bytes, resumable, "
|
||||
"req_segments, eff_segments, req_buffer_bytes, eff_buffer_bytes, queue_position, "
|
||||
"error_code, error_message, error_http_status, error_retryable, error_attempt, "
|
||||
"error_next_retry_at) VALUES("
|
||||
"error_next_retry_at, speed_bps) VALUES("
|
||||
"?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19,?20,?21,?22,"
|
||||
"?23,?24,?25,?26,?27,?28,?29,?30,?31,?32,?33)");
|
||||
"?23,?24,?25,?26,?27,?28,?29,?30,?31,?32,?33,?34)");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
|
||||
auto chk = [](DbResult<void> r) { return r.has_value(); };
|
||||
@@ -134,7 +135,8 @@ DbResult<void> Tasks::insert(const TaskRow& r) {
|
||||
chk(r.error_retryable ? st->bind(31, static_cast<std::int64_t>(*r.error_retryable))
|
||||
: st->bind_null(31)) &&
|
||||
chk(bind_opt(*st, 32, r.error_attempt)) &&
|
||||
chk(bind_opt(*st, 33, r.error_next_retry_at));
|
||||
chk(bind_opt(*st, 33, r.error_next_retry_at)) &&
|
||||
chk(st->bind(34, r.speed_bps));
|
||||
if (!ok) return std::unexpected(DbError{0, "failed to bind a task column"});
|
||||
|
||||
if (auto r2 = st->step(); !r2) return std::unexpected(r2.error());
|
||||
@@ -264,6 +266,50 @@ DbResult<bool> Tasks::remove(std::string_view task_id) {
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Tasks::update_progress(std::string_view task_id, std::int64_t downloaded_bytes,
|
||||
std::int64_t speed_bps, std::int64_t eff_segments,
|
||||
std::int64_t eff_buffer_bytes) {
|
||||
auto st = db_.prepare(
|
||||
"UPDATE tasks SET downloaded_bytes = ?2, speed_bps = ?3, eff_segments = ?4, "
|
||||
"eff_buffer_bytes = ?5 WHERE task_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto r = st->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(2, downloaded_bytes); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(3, speed_bps); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(4, eff_segments); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(5, eff_buffer_bytes); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Tasks::set_probe_result(std::string_view task_id, const ProbeFields& f) {
|
||||
auto st = db_.prepare(
|
||||
"UPDATE tasks SET size_bytes = ?2, resumable = ?3, etag = ?4, last_modified = ?5, "
|
||||
"content_type = ?6, effective_url = ?7 WHERE task_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto r = st->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = bind_opt(*st, 2, f.size_bytes); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(3, static_cast<std::int64_t>(f.resumable)); !r)
|
||||
return std::unexpected(r.error());
|
||||
if (auto r = bind_opt(*st, 4, f.etag); !r) return std::unexpected(r.error());
|
||||
if (auto r = bind_opt(*st, 5, f.last_modified); !r) return std::unexpected(r.error());
|
||||
if (auto r = bind_opt(*st, 6, f.content_type); !r) return std::unexpected(r.error());
|
||||
if (auto r = bind_opt(*st, 7, f.effective_url); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Tasks::set_final_bytes(std::string_view task_id, std::int64_t bytes) {
|
||||
auto st = db_.prepare(
|
||||
"UPDATE tasks SET downloaded_bytes = ?2, size_bytes = COALESCE(size_bytes, ?2) "
|
||||
"WHERE task_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto r = st->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(2, bytes); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<std::int64_t> Tasks::count() {
|
||||
auto st = db_.prepare("SELECT count(*) FROM tasks");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
@@ -272,6 +318,99 @@ DbResult<std::int64_t> Tasks::count() {
|
||||
return (*row) ? st->column_int(0) : 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Tasks::apply_update(std::string_view task_id, const UpdatePatch& patch) {
|
||||
// One UPDATE per present field: simplest thing that's obviously correct for a
|
||||
// single-row edit with ~9 independent optional fields, and it means a field the
|
||||
// caller didn't touch is never rewritten with its own unchanged value (matters for
|
||||
// no-op-detection callers, though download.update doesn't currently need that).
|
||||
bool touched_any = false;
|
||||
auto run = [&](const char* sql, auto&& binder) -> DbResult<void> {
|
||||
auto st = db_.prepare(sql);
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto r = st->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = binder(*st); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
touched_any = true;
|
||||
return {};
|
||||
};
|
||||
|
||||
if (patch.save_dir && patch.filename) {
|
||||
if (auto r = run("UPDATE tasks SET save_dir = ?2, filename = ?3 WHERE task_id = ?1",
|
||||
[&](Stmt& s) {
|
||||
if (auto b = s.bind(2, std::string_view(*patch.save_dir)); !b) return b;
|
||||
return s.bind(3, std::string_view(*patch.filename));
|
||||
});
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
if (patch.category_id) {
|
||||
if (auto r = run("UPDATE tasks SET category_id = ?2 WHERE task_id = ?1",
|
||||
[&](Stmt& s) { return s.bind(2, std::string_view(*patch.category_id)); });
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
if (patch.queue_id) {
|
||||
if (auto r = run("UPDATE tasks SET queue_id = ?2, queue_position = ?3 WHERE task_id = ?1",
|
||||
[&](Stmt& s) {
|
||||
if (auto b = s.bind(2, std::string_view(*patch.queue_id)); !b) return b;
|
||||
return patch.queue_position ? s.bind(3, *patch.queue_position)
|
||||
: s.bind_null(3);
|
||||
});
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
if (patch.description) {
|
||||
if (auto r = run("UPDATE tasks SET description = ?2 WHERE task_id = ?1",
|
||||
[&](Stmt& s) { return s.bind(2, std::string_view(*patch.description)); });
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
if (patch.req_segments) {
|
||||
if (auto r = run("UPDATE tasks SET req_segments = ?2 WHERE task_id = ?1",
|
||||
[&](Stmt& s) { return s.bind(2, *patch.req_segments); });
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
if (patch.req_buffer_bytes) {
|
||||
if (auto r = run("UPDATE tasks SET req_buffer_bytes = ?2 WHERE task_id = ?1",
|
||||
[&](Stmt& s) { return s.bind(2, *patch.req_buffer_bytes); });
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
if (patch.checksum_algo && patch.checksum_value) {
|
||||
if (auto r = run(
|
||||
"UPDATE tasks SET checksum_algo = ?2, checksum_value = ?3 WHERE task_id = ?1",
|
||||
[&](Stmt& s) {
|
||||
if (auto b = s.bind(2, std::string_view(*patch.checksum_algo)); !b) return b;
|
||||
return s.bind(3, std::string_view(*patch.checksum_value));
|
||||
});
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
|
||||
return touched_any;
|
||||
}
|
||||
|
||||
DbResult<bool> Tasks::set_url(std::string_view task_id, std::string_view url) {
|
||||
auto st = db_.prepare("UPDATE tasks SET url = ?2 WHERE task_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto r = st->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(2, url); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Tasks::has_active_duplicate(std::string_view url) {
|
||||
auto st = db_.prepare(
|
||||
"SELECT 1 FROM tasks WHERE url = ?1 "
|
||||
"AND state NOT IN ('complete','failed','cancelled') LIMIT 1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, url); !b) return std::unexpected(b.error());
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
return *row;
|
||||
}
|
||||
|
||||
proto::TaskSummary to_summary(const TaskRow& r) {
|
||||
proto::TaskSummary s;
|
||||
s.taskId = r.task_id;
|
||||
@@ -282,9 +421,16 @@ proto::TaskSummary to_summary(const TaskRow& r) {
|
||||
s.sizeBytes = r.size_bytes;
|
||||
s.downloadedBytes = r.downloaded_bytes;
|
||||
if (auto st = proto::parse_TaskState(r.state)) s.state = *st;
|
||||
s.speedBps = 0;
|
||||
s.speedBps = r.speed_bps;
|
||||
s.resumable = r.resumable;
|
||||
s.segments = r.eff_segments;
|
||||
// TaskSummary.segments is minimum:1, always -- even a task that has never connected
|
||||
// reports the count it WOULD use (its requested value, or the frozen default),
|
||||
// never the "not started yet" placeholder of 0 that used to leak onto the wire.
|
||||
s.segments = r.eff_segments > 0 ? r.eff_segments
|
||||
: r.req_segments && *r.req_segments > 0 ? *r.req_segments
|
||||
: 8;
|
||||
if (s.segments < 1) s.segments = 1;
|
||||
if (s.segments > 32) s.segments = 32;
|
||||
s.categoryId = r.category_id;
|
||||
s.queueId = r.queue_id;
|
||||
s.queuePosition = r.queue_position;
|
||||
|
||||
@@ -21,7 +21,7 @@ struct TaskRow {
|
||||
std::string save_dir;
|
||||
std::string filename;
|
||||
std::string state = "new";
|
||||
std::string start_mode = "auto";
|
||||
std::string start_mode = "now"; // contract StartMode: 'now'|'later'|'queue'
|
||||
std::string created_at;
|
||||
|
||||
std::optional<std::string> effective_url;
|
||||
@@ -39,6 +39,7 @@ struct TaskRow {
|
||||
|
||||
std::optional<std::int64_t> size_bytes;
|
||||
std::int64_t downloaded_bytes = 0;
|
||||
std::int64_t speed_bps = 0;
|
||||
bool resumable = false;
|
||||
|
||||
std::optional<std::int64_t> req_segments;
|
||||
@@ -78,6 +79,64 @@ public:
|
||||
DbResult<bool> remove(std::string_view task_id);
|
||||
DbResult<std::int64_t> count();
|
||||
|
||||
// capture.offer's dedupe check: true if a non-terminal task already targets this exact
|
||||
// URL (the same rule download.add itself does not enforce — a deliberate re-add is
|
||||
// allowed there; capture is the automatic path where re-grabbing an in-flight download
|
||||
// is almost always a mistake, e.g. two tabs triggering the same link).
|
||||
DbResult<bool> has_active_duplicate(std::string_view url);
|
||||
|
||||
// download.update's patch, already resolved by the caller (new save_dir/filename
|
||||
// canonicalized and root-checked, any file already moved on disk — this only writes
|
||||
// the row). Every field is applied when present; queue_position is written alongside
|
||||
// queue_id (nullopt leaves the existing position alone — the caller decides what
|
||||
// "moved into a queue" should set it to). Note: the generated parser collapses "field
|
||||
// absent" and "field explicitly null" to the same nullopt (DownloadUpdateParamsPatch
|
||||
// has no way to tell them apart on the wire as generated), so this — like the RPC
|
||||
// layer above it — can only ever set category_id/queue_id/description/checksum, never
|
||||
// clear them back to NULL through this call.
|
||||
struct UpdatePatch {
|
||||
std::optional<std::string> save_dir;
|
||||
std::optional<std::string> filename;
|
||||
std::optional<std::string> category_id;
|
||||
std::optional<std::string> queue_id;
|
||||
std::optional<std::int64_t> queue_position;
|
||||
std::optional<std::string> description;
|
||||
std::optional<std::int64_t> req_segments;
|
||||
std::optional<std::int64_t> req_buffer_bytes;
|
||||
std::optional<std::string> checksum_algo;
|
||||
std::optional<std::string> checksum_value;
|
||||
};
|
||||
DbResult<bool> apply_update(std::string_view task_id, const UpdatePatch& patch);
|
||||
|
||||
// download.refreshUrl: point the task at a freshly-issued URL. Separate from
|
||||
// apply_update/set_probe_result since neither owns the base `url` column — refreshUrl
|
||||
// is the one caller that changes it after creation.
|
||||
DbResult<bool> set_url(std::string_view task_id, std::string_view url);
|
||||
|
||||
// Byte-counter update from an engine progress tick — cheaper than a full row rewrite,
|
||||
// and keeps download.list / download.get current between state transitions.
|
||||
DbResult<bool> update_progress(std::string_view task_id, std::int64_t downloaded_bytes,
|
||||
std::int64_t speed_bps, std::int64_t eff_segments,
|
||||
std::int64_t eff_buffer_bytes);
|
||||
|
||||
// What the probe learned, persisted before start() so a task that completes before any
|
||||
// progress tick still reports a real sizeBytes / resumable (not the pre-probe default).
|
||||
struct ProbeFields {
|
||||
std::optional<std::int64_t> size_bytes;
|
||||
bool resumable = false;
|
||||
std::optional<std::string> etag;
|
||||
std::optional<std::string> last_modified;
|
||||
std::optional<std::string> content_type;
|
||||
std::optional<std::string> effective_url;
|
||||
};
|
||||
DbResult<bool> set_probe_result(std::string_view task_id, const ProbeFields& fields);
|
||||
|
||||
// on_finished's byte count, for a task that completes before any progress tick ever
|
||||
// ran (see AGENT-DAEMON review: the bug this closes). size_bytes is only filled in if
|
||||
// still unset — the probe's total_size is the more authoritative source when both
|
||||
// exist and happen to disagree (a chunked source with no declared length, say).
|
||||
DbResult<bool> set_final_bytes(std::string_view task_id, std::int64_t bytes);
|
||||
|
||||
private:
|
||||
Db& db_;
|
||||
};
|
||||
|
||||
@@ -20,4 +20,10 @@ veloxd_test(sched_window LIBS veloxd_sched)
|
||||
veloxd_test(sched_governor LIBS veloxd_sched)
|
||||
veloxd_test(safepath LIBS veloxd_fs)
|
||||
veloxd_test(store_tasks LIBS veloxd_store)
|
||||
veloxd_test(sched_scheduler LIBS veloxd_sched)
|
||||
veloxd_test(sched_scheduler LIBS veloxd_sched veloxd_rpc)
|
||||
veloxd_test(event_hub LIBS veloxd_rpc)
|
||||
veloxd_test(store_categories_queues LIBS veloxd_store)
|
||||
veloxd_test(single_instance LIBS veloxd_rpc)
|
||||
veloxd_test(dispatcher_settings LIBS veloxd_rpc veloxd_store)
|
||||
veloxd_test(capture_offer LIBS veloxd_rpc veloxd_store)
|
||||
veloxd_test(dispatcher_misc LIBS veloxd_rpc veloxd_store)
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
// capture.offer: excluded hosts, type/size filtering, rule matching, dedupe, take/ignore,
|
||||
// and — the point of this file — the 750 ms deadline actually gets enforced when the
|
||||
// store is slow, without a real sleep anywhere (a fake CaptureDataSource advances its own
|
||||
// clock instead).
|
||||
|
||||
#include <chrono>
|
||||
#include <string>
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/capture_data_source.hpp"
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "store/migrations.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "store/tasks.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
using namespace velox::daemon;
|
||||
namespace proto = velox::proto;
|
||||
|
||||
namespace {
|
||||
|
||||
// A controllable CaptureDataSource: every accessor returns a canned value, and `now()`
|
||||
// reads a clock the test (or a "slow" accessor) can jump forward instantly. No real time
|
||||
// ever passes — a test that jumps 2 real seconds still runs in microseconds.
|
||||
class FakeCaptureSource : public rpc::CaptureDataSource {
|
||||
public:
|
||||
std::chrono::steady_clock::time_point clock = std::chrono::steady_clock::now();
|
||||
bool enabled = true;
|
||||
std::vector<std::string> ext = {"mp4"};
|
||||
std::vector<std::string> mime;
|
||||
std::int64_t min_size = 0;
|
||||
std::vector<std::string> excluded;
|
||||
std::vector<vdm::rules::Rule> rules_;
|
||||
std::string category = "video";
|
||||
std::string save_dir = "~/Downloads/velox-capture-test";
|
||||
bool duplicate = false;
|
||||
|
||||
// Set to jump the clock forward by this much the next time the named accessor is
|
||||
// called — simulates "this particular store read took a long time."
|
||||
std::chrono::milliseconds slow_on_rules{0};
|
||||
std::chrono::milliseconds slow_on_duplicate{0};
|
||||
|
||||
std::chrono::steady_clock::time_point now() override { return clock; }
|
||||
bool capture_enabled() override { return enabled; }
|
||||
std::vector<std::string> monitored_extensions() override { return ext; }
|
||||
std::vector<std::string> monitored_mime_types() override { return mime; }
|
||||
std::int64_t min_size_bytes() override { return min_size; }
|
||||
std::vector<std::string> excluded_hosts() override { return excluded; }
|
||||
std::vector<vdm::rules::Rule> enabled_rules() override {
|
||||
clock += slow_on_rules;
|
||||
return rules_;
|
||||
}
|
||||
std::string guess_category_id(const std::string&) override { return category; }
|
||||
std::string category_save_dir(const std::string&) override { return save_dir; }
|
||||
std::string default_save_dir() override { return save_dir; }
|
||||
bool has_active_duplicate(const std::string&) override {
|
||||
clock += slow_on_duplicate;
|
||||
return duplicate;
|
||||
}
|
||||
};
|
||||
|
||||
proto::CaptureOfferParams offer(std::string url) {
|
||||
proto::CaptureOfferParams p;
|
||||
p.url = std::move(url);
|
||||
p.method = proto::CaptureOfferParamsMethod::GET;
|
||||
p.tabUrl = "https://example.com/page";
|
||||
p.filename = std::string("movie.mp4");
|
||||
p.contentType = std::string("video/mp4");
|
||||
p.contentLength = 5'000'000;
|
||||
return p;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void run() {
|
||||
auto db = store::Db::open(":memory:");
|
||||
CHECK(db.has_value());
|
||||
if (!db) return;
|
||||
CHECK(store::migrate_to_head(*db).has_value());
|
||||
|
||||
rpc::EventHub hub;
|
||||
rpc::VeloxDispatcher dispatcher(*db, hub);
|
||||
|
||||
// --- capture disabled --------------------------------------------------------------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.enabled = false;
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK(r->action == proto::CaptureOfferResultAction::Ignore);
|
||||
CHECK(r->reason == proto::CaptureOfferResultReason::CaptureDisabled);
|
||||
}
|
||||
}
|
||||
|
||||
// --- excluded host -------------------------------------------------------------------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.excluded = {"*.excluded.example"};
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.excluded.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK(r->action == proto::CaptureOfferResultAction::Ignore);
|
||||
CHECK(r->reason == proto::CaptureOfferResultReason::ExcludedHost);
|
||||
}
|
||||
}
|
||||
|
||||
// --- type not monitored --------------------------------------------------------------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.ext = {"iso"}; // movie.mp4 doesn't match, and mime list is empty
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) CHECK(r->reason == proto::CaptureOfferResultReason::TypeNotMonitored);
|
||||
}
|
||||
|
||||
// --- below minimum size ----------------------------------------------------------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.min_size = 10'000'000; // offer's contentLength is 5,000,000
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) CHECK(r->reason == proto::CaptureOfferResultReason::BelowMinSize);
|
||||
}
|
||||
|
||||
// --- a rule says ignore this host --------------------------------------------------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
vdm::rules::Rule rule;
|
||||
rule.rule_id = "r1";
|
||||
rule.enabled = true;
|
||||
rule.priority = 0;
|
||||
rule.match.host_pattern = "cdn.example";
|
||||
rule.action.capture = vdm::rules::CaptureVerdict::ignore;
|
||||
src.rules_ = {rule};
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) CHECK(r->reason == proto::CaptureOfferResultReason::RuleIgnore);
|
||||
}
|
||||
|
||||
// --- duplicate -----------------------------------------------------------------------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.duplicate = true;
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) CHECK(r->reason == proto::CaptureOfferResultReason::Duplicate);
|
||||
}
|
||||
|
||||
// --- take: a real task gets created, saved under the resolved category dir --------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK(r->action == proto::CaptureOfferResultAction::Take);
|
||||
CHECK(r->taskId.has_value());
|
||||
if (r->taskId) {
|
||||
store::Tasks tasks(*db);
|
||||
auto row = tasks.get(*r->taskId);
|
||||
CHECK(row.has_value() && row->has_value());
|
||||
if (row && *row) {
|
||||
// resolve_target expands "~" and returns the canonical absolute path,
|
||||
// not the literal string handed in.
|
||||
CHECK((*row)->save_dir.find("velox-capture-test") != std::string::npos);
|
||||
CHECK_EQ((*row)->category_id.value_or(""), std::string("video"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- the deadline: a "slow" store still gets an answer, and it's `ignore` --------
|
||||
// slow_on_rules jumps the fake clock forward 2 real seconds' worth the moment
|
||||
// enabled_rules() is read (simulating a slow rules-table read); the 700 ms budget is
|
||||
// long blown by the time the next checkpoint runs, so the offer must be ignored
|
||||
// rather than proceeding to actually take the download. No real time passes — this
|
||||
// whole test runs in microseconds.
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.slow_on_rules = std::chrono::milliseconds(2000);
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
const auto wall_start = std::chrono::steady_clock::now();
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
const auto wall_elapsed = std::chrono::steady_clock::now() - wall_start;
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK(r->action == proto::CaptureOfferResultAction::Ignore);
|
||||
CHECK(!r->taskId.has_value());
|
||||
}
|
||||
// The real wall clock barely moved — only the fake one jumped — proving the
|
||||
// deadline check reads the injected clock, not a real sleep standing in for one.
|
||||
CHECK(wall_elapsed < std::chrono::milliseconds(500));
|
||||
}
|
||||
|
||||
// Same again, but the slow step is the dedupe check instead of rule matching — the
|
||||
// deadline check has to run between every step, not just after one particular call.
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.slow_on_duplicate = std::chrono::milliseconds(2000);
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) CHECK(r->action == proto::CaptureOfferResultAction::Ignore);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
@@ -0,0 +1,244 @@
|
||||
// rules.list/upsert, queue.reorder, schedule.get/set, limiter.get/set, download.update:
|
||||
// the rest of D3, called directly against an in-memory store (no socket).
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "store/migrations.hpp"
|
||||
#include "store/queues.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "store/tasks.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
using namespace velox::daemon;
|
||||
namespace proto = velox::proto;
|
||||
|
||||
namespace {
|
||||
|
||||
store::TaskRow task(std::string id, std::optional<std::string> queue = std::nullopt,
|
||||
std::int64_t pos = 0) {
|
||||
store::TaskRow r;
|
||||
r.task_id = std::move(id);
|
||||
r.url = "https://cdn.example/" + r.task_id;
|
||||
r.save_dir = "/tmp";
|
||||
r.filename = r.task_id + ".bin";
|
||||
r.state = "queued";
|
||||
r.created_at = "2026-09-12T00:00:00Z";
|
||||
r.queue_id = std::move(queue);
|
||||
if (r.queue_id) r.queue_position = pos;
|
||||
return r;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void run() {
|
||||
auto db = store::Db::open(":memory:");
|
||||
CHECK(db.has_value());
|
||||
if (!db) return;
|
||||
CHECK(store::migrate_to_head(*db).has_value());
|
||||
|
||||
rpc::EventHub hub;
|
||||
rpc::VeloxDispatcher dispatcher(*db, hub);
|
||||
|
||||
// --- rules.list starts empty; rules.upsert creates, replaces, and removes --------
|
||||
{
|
||||
auto listed = dispatcher.on_rules_list({});
|
||||
CHECK(listed.has_value());
|
||||
if (listed) CHECK(listed->items.empty());
|
||||
|
||||
proto::RulesUpsertParams up;
|
||||
proto::Rule r;
|
||||
r.name = "ISOs to a queue";
|
||||
r.enabled = true;
|
||||
r.priority = 5;
|
||||
r.match.extensions = std::vector<std::string>{"iso"};
|
||||
r.action.categoryId = "programs";
|
||||
up.upsert = {r};
|
||||
auto created = dispatcher.on_rules_upsert(up);
|
||||
CHECK(created.has_value());
|
||||
if (created) {
|
||||
CHECK_EQ(created->items.size(), std::size_t{1});
|
||||
CHECK(!created->items[0].ruleId.empty());
|
||||
}
|
||||
|
||||
// Replace: same ruleId, new priority.
|
||||
if (!created || created->items.empty()) return;
|
||||
std::string rule_id = created->items[0].ruleId;
|
||||
proto::RulesUpsertParams replace;
|
||||
proto::Rule r2 = created->items[0];
|
||||
r2.priority = 1;
|
||||
replace.upsert = {r2};
|
||||
auto replaced = dispatcher.on_rules_upsert(replace);
|
||||
CHECK(replaced.has_value());
|
||||
if (replaced) {
|
||||
CHECK_EQ(replaced->items.size(), std::size_t{1});
|
||||
CHECK_EQ(replaced->items[0].priority, std::int64_t{1});
|
||||
}
|
||||
|
||||
// Remove.
|
||||
proto::RulesUpsertParams rm;
|
||||
rm.remove = {rule_id};
|
||||
auto removed = dispatcher.on_rules_upsert(rm);
|
||||
CHECK(removed.has_value());
|
||||
if (removed) CHECK(removed->items.empty());
|
||||
}
|
||||
|
||||
// --- queue.reorder: a real permutation applies; a non-permutation is -32602 ------
|
||||
{
|
||||
store::Tasks tasks(*db);
|
||||
CHECK(tasks.insert(task("q0", "main", 0)).has_value());
|
||||
CHECK(tasks.insert(task("q1", "main", 1)).has_value());
|
||||
CHECK(tasks.insert(task("q2", "main", 2)).has_value());
|
||||
|
||||
proto::QueueReorderParams p;
|
||||
p.queueId = "main";
|
||||
p.taskIds = {"q2", "q0", "q1"};
|
||||
auto r = dispatcher.on_queue_reorder(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK(r->queue.taskIds.has_value());
|
||||
if (r->queue.taskIds) {
|
||||
const auto& ids = *r->queue.taskIds;
|
||||
CHECK_EQ(ids.size(), std::size_t{3});
|
||||
if (ids.size() == 3) {
|
||||
CHECK_EQ(ids[0], std::string("q2"));
|
||||
CHECK_EQ(ids[1], std::string("q0"));
|
||||
CHECK_EQ(ids[2], std::string("q1"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
proto::QueueReorderParams bad;
|
||||
bad.queueId = "main";
|
||||
bad.taskIds = {"q0", "q1"}; // missing q2: not a permutation
|
||||
auto bad_r = dispatcher.on_queue_reorder(bad);
|
||||
CHECK(!bad_r.has_value());
|
||||
if (!bad_r) CHECK(bad_r.error().code == proto::ErrorCode::InvalidParams);
|
||||
|
||||
proto::QueueReorderParams unknown_queue;
|
||||
unknown_queue.queueId = "does-not-exist";
|
||||
unknown_queue.taskIds = {};
|
||||
auto unknown_r = dispatcher.on_queue_reorder(unknown_queue);
|
||||
CHECK(!unknown_r.has_value());
|
||||
|
||||
tasks.remove("q0");
|
||||
tasks.remove("q1");
|
||||
tasks.remove("q2");
|
||||
}
|
||||
|
||||
// --- schedule.get / schedule.set ---------------------------------------------------
|
||||
{
|
||||
proto::ScheduleGetParams g;
|
||||
g.queueId = std::string("main");
|
||||
auto before = dispatcher.on_schedule_get(g);
|
||||
CHECK(before.has_value());
|
||||
if (before) {
|
||||
CHECK_EQ(before->items.size(), std::size_t{1});
|
||||
if (!before->items.empty()) CHECK(!before->items[0].schedule.has_value());
|
||||
}
|
||||
|
||||
proto::ScheduleSetParams set;
|
||||
set.queueId = "main";
|
||||
proto::Schedule sched;
|
||||
sched.enabled = true;
|
||||
sched.mode = proto::ScheduleMode::Periodic;
|
||||
sched.startTime = std::string("22:00");
|
||||
sched.stopTime = std::string("06:00");
|
||||
set.schedule = sched;
|
||||
auto set_r = dispatcher.on_schedule_set(set);
|
||||
CHECK(set_r.has_value());
|
||||
if (set_r) {
|
||||
CHECK(set_r->schedule.has_value());
|
||||
CHECK(!set_r->nextRunAt.has_value()); // documented gap, not computed
|
||||
}
|
||||
|
||||
auto after = dispatcher.on_schedule_get(g);
|
||||
CHECK(after.has_value());
|
||||
if (after && !after->items.empty())
|
||||
CHECK(after->items[0].schedule.has_value());
|
||||
|
||||
// Clearing (the wire-level "explicit null" is unreachable through the generated
|
||||
// parser's collapsing of absent/null — see UpdatePatch's own comment; this test
|
||||
// only proves set-a-value works, not clear).
|
||||
|
||||
proto::ScheduleGetParams all;
|
||||
auto all_r = dispatcher.on_schedule_get(all);
|
||||
CHECK(all_r.has_value());
|
||||
if (all_r) CHECK(!all_r->items.empty());
|
||||
|
||||
proto::ScheduleGetParams missing;
|
||||
missing.queueId = std::string("does-not-exist");
|
||||
auto missing_r = dispatcher.on_schedule_get(missing);
|
||||
CHECK(missing_r.has_value());
|
||||
if (missing_r) CHECK(missing_r->items.empty());
|
||||
}
|
||||
|
||||
// --- limiter.get / limiter.set -----------------------------------------------------
|
||||
{
|
||||
auto before = dispatcher.on_limiter_get({});
|
||||
CHECK(before.has_value());
|
||||
if (before) {
|
||||
CHECK(!before->enabled);
|
||||
CHECK_EQ(before->globalBps, std::int64_t{0});
|
||||
}
|
||||
|
||||
proto::Limiter set;
|
||||
set.enabled = true;
|
||||
set.globalBps = 2'000'000;
|
||||
auto set_r = dispatcher.on_limiter_set(set);
|
||||
CHECK(set_r.has_value());
|
||||
if (set_r) {
|
||||
CHECK(set_r->enabled);
|
||||
CHECK_EQ(set_r->globalBps, std::int64_t{2'000'000});
|
||||
}
|
||||
|
||||
auto after = dispatcher.on_limiter_get({});
|
||||
CHECK(after.has_value());
|
||||
if (after) {
|
||||
CHECK(after->enabled);
|
||||
CHECK_EQ(after->globalBps, std::int64_t{2'000'000});
|
||||
}
|
||||
}
|
||||
|
||||
// --- download.update: metadata fields, range validation, not-found ---------------
|
||||
{
|
||||
store::Tasks tasks(*db);
|
||||
CHECK(tasks.insert(task("u0")).has_value());
|
||||
|
||||
proto::DownloadUpdateParams p;
|
||||
p.taskId = "u0";
|
||||
p.patch.description = std::string("a note");
|
||||
p.patch.segments = 4;
|
||||
auto r = dispatcher.on_download_update(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK_EQ(r->taskId, std::string("u0"));
|
||||
}
|
||||
auto row = tasks.get("u0");
|
||||
CHECK(row.has_value() && row->has_value());
|
||||
if (row && *row) {
|
||||
CHECK_EQ((*row)->description.value_or(""), std::string("a note"));
|
||||
CHECK_EQ((*row)->req_segments.value_or(-1), std::int64_t{4});
|
||||
}
|
||||
|
||||
proto::DownloadUpdateParams bad_range;
|
||||
bad_range.taskId = "u0";
|
||||
bad_range.patch.segments = 999;
|
||||
auto bad_r = dispatcher.on_download_update(bad_range);
|
||||
CHECK(!bad_r.has_value());
|
||||
if (!bad_r) CHECK(bad_r.error().code == proto::ErrorCode::InvalidParams);
|
||||
|
||||
proto::DownloadUpdateParams missing;
|
||||
missing.taskId = "does-not-exist";
|
||||
missing.patch.description = std::string("x");
|
||||
auto missing_r = dispatcher.on_download_update(missing);
|
||||
CHECK(!missing_r.has_value());
|
||||
if (!missing_r) CHECK(missing_r.error().code == proto::ErrorCode::TaskNotFound);
|
||||
|
||||
tasks.remove("u0");
|
||||
}
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
@@ -0,0 +1,142 @@
|
||||
// settings.get / settings.set: the field <-> SettingKey <-> JSON-type mapping table in
|
||||
// dispatcher.cpp, called directly (no socket) against an in-memory store.
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "store/migrations.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
using namespace velox::daemon;
|
||||
namespace proto = velox::proto;
|
||||
|
||||
void run() {
|
||||
auto db = store::Db::open(":memory:");
|
||||
CHECK(db.has_value());
|
||||
if (!db) return;
|
||||
CHECK(store::migrate_to_head(*db).has_value());
|
||||
|
||||
rpc::EventHub hub;
|
||||
rpc::VeloxDispatcher dispatcher(*db, hub);
|
||||
|
||||
// --- get with keys=nullopt: every one of the 43 SettingKeys comes back -----------
|
||||
{
|
||||
proto::SettingsGetParams p;
|
||||
auto r = dispatcher.on_settings_get(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
int present = 0;
|
||||
present += r->values.general_launchOnLogin.has_value();
|
||||
present += r->values.capture_enabled.has_value();
|
||||
present += r->values.saveTo_allowedRoots.has_value();
|
||||
present += r->values.connection_maxConcurrentDownloads.has_value();
|
||||
present += r->values.proxy_mode.has_value();
|
||||
present += r->values.sounds_onError.has_value();
|
||||
CHECK_EQ(present, 6);
|
||||
// A default, sight-checked: connection.maxConcurrentDownloads is 5 (settings.cpp).
|
||||
CHECK_EQ(r->values.connection_maxConcurrentDownloads.value_or(-1), std::int64_t{5});
|
||||
CHECK(r->values.saveTo_allowedRoots.has_value());
|
||||
if (r->values.saveTo_allowedRoots)
|
||||
CHECK_EQ(r->values.saveTo_allowedRoots->size(), std::size_t{1});
|
||||
// Enum-typed default round-trips through parse_XXX correctly.
|
||||
CHECK(r->values.proxy_mode == proto::SettingsProxyMode::System);
|
||||
}
|
||||
}
|
||||
|
||||
// --- get with an explicit key list: only those fields are populated --------------
|
||||
{
|
||||
proto::SettingsGetParams p;
|
||||
p.keys = {proto::SettingKey::GeneralLaunchOnLogin,
|
||||
proto::SettingKey::ConnectionMaxConcurrentDownloads};
|
||||
auto r = dispatcher.on_settings_get(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK(r->values.general_launchOnLogin.has_value());
|
||||
CHECK(r->values.connection_maxConcurrentDownloads.has_value());
|
||||
CHECK(!r->values.capture_enabled.has_value());
|
||||
CHECK(!r->values.proxy_mode.has_value());
|
||||
}
|
||||
}
|
||||
|
||||
// --- set: a valid change is persisted, reported in `changed`, and readable back ---
|
||||
{
|
||||
proto::SettingsSetParams p;
|
||||
p.values.connection_maxConcurrentDownloads = 12;
|
||||
p.values.general_launchOnLogin = true;
|
||||
auto r = dispatcher.on_settings_set(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK_EQ(r->changed.size(), std::size_t{2});
|
||||
CHECK_EQ(r->values.connection_maxConcurrentDownloads.value_or(-1), std::int64_t{12});
|
||||
CHECK_EQ(r->values.general_launchOnLogin.value_or(false), true);
|
||||
}
|
||||
|
||||
proto::SettingsGetParams g;
|
||||
g.keys = {proto::SettingKey::ConnectionMaxConcurrentDownloads};
|
||||
auto g_r = dispatcher.on_settings_get(g);
|
||||
CHECK(g_r.has_value());
|
||||
if (g_r) CHECK_EQ(g_r->values.connection_maxConcurrentDownloads.value_or(-1), std::int64_t{12});
|
||||
}
|
||||
|
||||
// --- set: setting the same value again reports it unchanged ----------------------
|
||||
{
|
||||
proto::SettingsSetParams p;
|
||||
p.values.connection_maxConcurrentDownloads = 12;
|
||||
auto r = dispatcher.on_settings_set(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) CHECK(r->changed.empty());
|
||||
}
|
||||
|
||||
// --- set: out of range is rejected, and nothing else in the same call lands ------
|
||||
{
|
||||
proto::SettingsSetParams p;
|
||||
p.values.connection_maxConcurrentDownloads = 999; // max 64
|
||||
p.values.general_minimizeToTray = true; // would otherwise succeed
|
||||
auto r = dispatcher.on_settings_set(p);
|
||||
CHECK(!r.has_value());
|
||||
if (!r) CHECK(r.error().code == proto::ErrorCode::InvalidParams);
|
||||
|
||||
proto::SettingsGetParams g;
|
||||
g.keys = {proto::SettingKey::GeneralMinimizeToTray};
|
||||
auto g_r = dispatcher.on_settings_get(g);
|
||||
CHECK(g_r.has_value());
|
||||
// Rejected as a whole: minimizeToTray was never written even though its own value
|
||||
// was valid on its own.
|
||||
if (g_r) CHECK_EQ(g_r->values.general_minimizeToTray.value_or(true), false);
|
||||
}
|
||||
|
||||
// --- set: an enum field round-trips ------------------------------------------------
|
||||
{
|
||||
proto::SettingsSetParams p;
|
||||
p.values.proxy_mode = proto::SettingsProxyMode::Socks5;
|
||||
auto r = dispatcher.on_settings_set(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK_EQ(r->changed.size(), std::size_t{1});
|
||||
CHECK(r->values.proxy_mode == proto::SettingsProxyMode::Socks5);
|
||||
}
|
||||
}
|
||||
|
||||
// --- set: saveTo.defaultDir outside every allowed root is rejected with -32011 ----
|
||||
{
|
||||
proto::SettingsSetParams p;
|
||||
p.values.saveTo_defaultDir = "/definitely/not/an/allowed/root";
|
||||
auto r = dispatcher.on_settings_set(p);
|
||||
CHECK(!r.has_value());
|
||||
if (!r) CHECK(r.error().code == proto::ErrorCode::InvalidPath);
|
||||
}
|
||||
|
||||
// --- set: saveTo.allowedRoots with an unresolvable entry is rejected with -32011 --
|
||||
{
|
||||
proto::SettingsSetParams p;
|
||||
p.values.saveTo_allowedRoots = {"/this/path/should/never/exist/anywhere"};
|
||||
auto r = dispatcher.on_settings_set(p);
|
||||
CHECK(!r.has_value());
|
||||
if (!r) CHECK(r.error().code == proto::ErrorCode::InvalidPath);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
@@ -0,0 +1,68 @@
|
||||
#include "rpc/event_hub.hpp"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "check.hpp"
|
||||
|
||||
using velox::daemon::rpc::EventHub;
|
||||
namespace proto = velox::proto;
|
||||
|
||||
void run() {
|
||||
EventHub hub;
|
||||
|
||||
// --- no subscribers: publish is a no-op, not an error --------------------------
|
||||
hub.publish(proto::Event::TaskAdded, nlohmann::json{{"x", 1}});
|
||||
|
||||
// --- a subscription with no filter set receives nothing -------------------
|
||||
std::vector<nlohmann::json> a;
|
||||
const auto sub_a = hub.subscribe([&](const nlohmann::json& n) { a.push_back(n); });
|
||||
hub.publish(proto::Event::TaskAdded, nlohmann::json{{"n", 1}});
|
||||
CHECK_EQ(a.size(), 0u);
|
||||
|
||||
// --- filtering by event kind ------------------------------------------------
|
||||
hub.set_filter(sub_a, {proto::Event::TaskAdded}, std::nullopt);
|
||||
hub.publish(proto::Event::TaskAdded, nlohmann::json{{"n", 2}});
|
||||
hub.publish(proto::Event::TaskRemoved, nlohmann::json{{"n", 3}}); // not subscribed
|
||||
CHECK_EQ(a.size(), 1u);
|
||||
CHECK_EQ(a[0]["n"].get<int>(), 2);
|
||||
|
||||
// --- two subscribers, independent filters -----------------------------------
|
||||
std::vector<nlohmann::json> b;
|
||||
const auto sub_b = hub.subscribe([&](const nlohmann::json& n) { b.push_back(n); });
|
||||
hub.set_filter(sub_b, {proto::Event::TaskRemoved}, std::nullopt);
|
||||
hub.publish(proto::Event::TaskAdded, nlohmann::json{{"n", 4}});
|
||||
hub.publish(proto::Event::TaskRemoved, nlohmann::json{{"n", 5}});
|
||||
CHECK_EQ(a.size(), 2u); // got the TaskAdded
|
||||
CHECK_EQ(b.size(), 1u); // got the TaskRemoved
|
||||
CHECK_EQ(b[0]["n"].get<int>(), 5);
|
||||
|
||||
// --- per-task filter: only the named ids reach the subscriber ---------------
|
||||
std::vector<nlohmann::json> c;
|
||||
const auto sub_c = hub.subscribe([&](const nlohmann::json& n) { c.push_back(n); });
|
||||
hub.set_filter(sub_c, {proto::Event::TaskState},
|
||||
std::vector<std::string>{"t1", "t2"});
|
||||
hub.publish(proto::Event::TaskState, nlohmann::json{{"n", 6}}, "t1");
|
||||
hub.publish(proto::Event::TaskState, nlohmann::json{{"n", 7}}, "t9"); // filtered out
|
||||
hub.publish(proto::Event::TaskState, nlohmann::json{{"n", 8}}, "t2");
|
||||
CHECK_EQ(c.size(), 2u);
|
||||
CHECK_EQ(c[0]["n"].get<int>(), 6);
|
||||
CHECK_EQ(c[1]["n"].get<int>(), 8);
|
||||
|
||||
// --- a non-task-scoped publish (no task_id) reaches a task-filtered sub too --
|
||||
// (matches "session.subscribe narrows task events" — a global event isn't one)
|
||||
hub.publish(proto::Event::TaskState, nlohmann::json{{"n", 9}});
|
||||
CHECK_EQ(c.size(), 3u);
|
||||
|
||||
// --- set_filter replaces, it does not add -----------------------------------
|
||||
hub.set_filter(sub_a, {proto::Event::TaskRemoved}, std::nullopt);
|
||||
hub.publish(proto::Event::TaskAdded, nlohmann::json{{"n", 10}});
|
||||
CHECK_EQ(a.size(), 2u); // still 2 -- TaskAdded no longer reaches sub_a
|
||||
|
||||
// --- unsubscribe stops delivery ------------------------------------------
|
||||
hub.unsubscribe(sub_b);
|
||||
hub.publish(proto::Event::TaskRemoved, nlohmann::json{{"n", 11}});
|
||||
CHECK_EQ(b.size(), 1u); // unchanged
|
||||
CHECK_EQ(a.size(), 3u); // sub_a still gets TaskRemoved
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
@@ -3,7 +3,10 @@
|
||||
|
||||
#include <string>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "sched/fake_engine_port.hpp"
|
||||
#include "sched/governor.hpp"
|
||||
#include "sched/scheduler.hpp"
|
||||
@@ -91,10 +94,10 @@ void run() {
|
||||
CHECK_EQ(engine.starts.size(), 1u); // b0 only
|
||||
CHECK_EQ(task_state(*db, "b0"), std::string("probing"));
|
||||
|
||||
sched.on_engine_state("b0", "downloading", std::nullopt);
|
||||
sched.on_engine_state("b0", "probing", "downloading", std::nullopt);
|
||||
CHECK_EQ(task_state(*db, "b0"), std::string("downloading"));
|
||||
|
||||
sched.on_engine_state("b0", "complete", std::nullopt);
|
||||
sched.on_engine_state("b0", "downloading", "complete", std::nullopt);
|
||||
CHECK_EQ(task_state(*db, "b0"), std::string("complete"));
|
||||
|
||||
CHECK(sched.tick().has_value()); // b0 terminal -> b1 admitted
|
||||
@@ -113,7 +116,7 @@ void run() {
|
||||
CHECK(tasks.insert(task("q0", "queued", "2026-09-10T10:00:00Z", "main", 0)).has_value());
|
||||
CHECK(sched.tick().has_value());
|
||||
CHECK_EQ(engine.starts.size(), 1u);
|
||||
sched.on_engine_state("q0", "downloading", std::nullopt);
|
||||
sched.on_engine_state("q0", "connecting", "downloading", std::nullopt);
|
||||
|
||||
CHECK(db->exec("UPDATE queues SET state='stopped' WHERE queue_id='main'").has_value());
|
||||
CHECK(sched.tick().has_value());
|
||||
@@ -143,7 +146,7 @@ void run() {
|
||||
ef.code = "auth_required";
|
||||
ef.message = "401";
|
||||
ef.http_status = 401;
|
||||
sched.on_engine_state("auth", "paused", ef);
|
||||
sched.on_engine_state("auth", "connecting", "paused", ef);
|
||||
|
||||
store::Tasks t(*db);
|
||||
auto row = t.get("auth").value().value();
|
||||
@@ -186,6 +189,292 @@ void run() {
|
||||
CHECK_EQ(engine.max_active_segments.back(), 12u);
|
||||
CHECK_EQ(sched.reload_config().has_value() ? 0 : 1, 0);
|
||||
}
|
||||
|
||||
// --- event.task.state: published on admission, on a paused transition, and on an
|
||||
// engine-reported transition; previousState reflects the store's prior row ----------
|
||||
{
|
||||
for (const char* id : {"r0", "r1", "r2"}) tasks.remove(id); // leftover from above
|
||||
FakeEnginePort engine;
|
||||
rpc::EventHub hub;
|
||||
Scheduler sched(*db, engine, Governor(GovernorConfig{.max_concurrent_downloads = 10,
|
||||
.max_active_segments = 32}),
|
||||
&hub);
|
||||
std::vector<nlohmann::json> received;
|
||||
const auto sub = hub.subscribe([&](const nlohmann::json& n) { received.push_back(n); });
|
||||
hub.set_filter(sub, {velox::proto::Event::TaskState}, std::nullopt);
|
||||
|
||||
CHECK(tasks.insert(task("ev0", "queued", "2026-09-10T10:00:00Z")).has_value());
|
||||
CHECK(sched.tick().has_value()); // admits ev0: queued -> probing
|
||||
CHECK_EQ(received.size(), 1u);
|
||||
CHECK_EQ(received[0]["params"]["taskId"].get<std::string>(), std::string("ev0"));
|
||||
CHECK_EQ(received[0]["params"]["state"].get<std::string>(), std::string("probing"));
|
||||
CHECK_EQ(received[0]["params"]["previousState"].get<std::string>(), std::string("queued"));
|
||||
CHECK(received[0]["params"]["error"].is_null());
|
||||
CHECK_EQ(received[0]["params"]["summary"]["taskId"].get<std::string>(), std::string("ev0"));
|
||||
|
||||
sched.on_engine_state("ev0", "probing", "downloading", std::nullopt);
|
||||
CHECK_EQ(received.size(), 2u);
|
||||
CHECK_EQ(received[1]["params"]["previousState"].get<std::string>(), std::string("probing"));
|
||||
CHECK_EQ(received[1]["params"]["state"].get<std::string>(), std::string("downloading"));
|
||||
|
||||
sched::TaskErrorFields ef;
|
||||
ef.code = "connection_reset";
|
||||
ef.message = "reset";
|
||||
sched.on_engine_state("ev0", "downloading", "paused", ef);
|
||||
CHECK_EQ(received.size(), 3u);
|
||||
CHECK(!received[2]["params"]["error"].is_null());
|
||||
CHECK_EQ(received[2]["params"]["error"]["code"].get<std::string>(),
|
||||
std::string("connection_reset"));
|
||||
|
||||
tasks.remove("ev0");
|
||||
}
|
||||
|
||||
// --- progress_snapshot: only started tasks, plus a store side-effect --------------
|
||||
{
|
||||
FakeEnginePort engine;
|
||||
Scheduler sched(*db, engine,
|
||||
Governor(GovernorConfig{.max_concurrent_downloads = 10,
|
||||
.max_active_segments = 32}));
|
||||
CHECK(tasks.insert(task("pr0", "queued", "2026-09-10T10:00:00Z")).has_value());
|
||||
CHECK(sched.tick().has_value());
|
||||
CHECK_EQ(engine.starts.size(), 1u);
|
||||
|
||||
vdm::task::Progress p;
|
||||
p.downloaded = 12345;
|
||||
p.speed_bps = 999;
|
||||
p.effective_segments = 4;
|
||||
p.effective_buffer_bytes = 65536;
|
||||
engine.fake_progress[engine.starts[0].id.value] = p;
|
||||
|
||||
const auto snap = sched.progress_snapshot();
|
||||
CHECK_EQ(snap.size(), 1u);
|
||||
CHECK_EQ(snap[0].task_id, std::string("pr0"));
|
||||
CHECK_EQ(snap[0].downloaded_bytes, 12345u);
|
||||
CHECK_EQ(snap[0].speed_bps, 999u);
|
||||
|
||||
auto row = tasks.get("pr0").value().value();
|
||||
CHECK_EQ(row.downloaded_bytes, 12345);
|
||||
CHECK_EQ(row.eff_segments, 4);
|
||||
CHECK((row.eff_buffer_bytes.has_value() && *row.eff_buffer_bytes == 65536));
|
||||
|
||||
tasks.remove("pr0");
|
||||
}
|
||||
|
||||
// --- user_pause / user_resume: a live task, and one that never started -----------
|
||||
{
|
||||
FakeEnginePort engine;
|
||||
Scheduler sched(*db, engine,
|
||||
Governor(GovernorConfig{.max_concurrent_downloads = 10,
|
||||
.max_active_segments = 32}));
|
||||
CHECK(tasks.insert(task("up0", "queued", "2026-09-10T10:00:00Z")).has_value());
|
||||
CHECK(tasks.insert(task("up1", "paused", "2026-09-10T10:00:00Z")).has_value());
|
||||
CHECK(sched.tick().has_value()); // admits up0; up1 stays paused (governor never
|
||||
// touches a user-owned pause)
|
||||
CHECK_EQ(engine.starts.size(), 1u);
|
||||
const auto live_id = engine.starts[0].id;
|
||||
|
||||
// Pausing a live task calls the engine now and transitions eagerly — not left for
|
||||
// the next tick.
|
||||
auto r = sched.user_pause("up0");
|
||||
CHECK(r.found);
|
||||
CHECK(r.changed);
|
||||
CHECK_EQ(r.state, std::string("paused"));
|
||||
CHECK_EQ(engine.paused.size(), 1u);
|
||||
CHECK_EQ(engine.paused[0].value, live_id.value);
|
||||
CHECK_EQ(task_state(*db, "up0"), std::string("paused"));
|
||||
auto row = tasks.get("up0").value().value();
|
||||
CHECK_EQ(row.pause_reason.value_or(""), std::string("user"));
|
||||
|
||||
// Idempotent: pausing an already-paused task is a no-op, not an error.
|
||||
auto again = sched.user_pause("up0");
|
||||
CHECK(again.found);
|
||||
CHECK(!again.changed);
|
||||
|
||||
// Resuming a task that still holds a live engine handle calls engine.resume() and
|
||||
// goes straight to `connecting`.
|
||||
auto res = sched.user_resume("up0");
|
||||
CHECK(res.found);
|
||||
CHECK(res.changed);
|
||||
CHECK_EQ(res.state, std::string("connecting"));
|
||||
CHECK_EQ(engine.resumed.size(), 1u);
|
||||
CHECK_EQ(engine.resumed[0].value, live_id.value);
|
||||
|
||||
// The engine's own delayed pause-ack (on_state with no error, arriving after the
|
||||
// eager transition already wrote the real reason) must not clobber pause_reason
|
||||
// back to NULL.
|
||||
(void)sched.user_pause("up0");
|
||||
sched.on_engine_state("up0", "downloading", "paused", std::nullopt);
|
||||
auto row2 = tasks.get("up0").value().value();
|
||||
CHECK_EQ(row2.pause_reason.value_or(""), std::string("user"));
|
||||
|
||||
// up1 never started (still parked, no engine handle): resume just re-queues it for
|
||||
// the next tick's normal admission.
|
||||
auto res2 = sched.user_resume("up1");
|
||||
CHECK(res2.found);
|
||||
CHECK(res2.changed);
|
||||
CHECK_EQ(res2.state, std::string("queued"));
|
||||
CHECK(engine.resumed.size() == 1u); // up1 was never mapped; no engine call
|
||||
|
||||
// Not found: a bogus id reports found=false, not a crash.
|
||||
auto missing = sched.user_pause("does-not-exist");
|
||||
CHECK(!missing.found);
|
||||
|
||||
tasks.remove("up0");
|
||||
tasks.remove("up1");
|
||||
}
|
||||
|
||||
// --- user_cancel + pause_queue --------------------------------------------------
|
||||
{
|
||||
CHECK(db->exec("UPDATE queues SET state='running' WHERE queue_id='main'").has_value());
|
||||
FakeEnginePort engine;
|
||||
Scheduler sched(*db, engine,
|
||||
Governor(GovernorConfig{.max_concurrent_downloads = 10,
|
||||
.max_active_segments = 32}));
|
||||
CHECK(tasks.insert(task("uc0", "queued", "2026-09-10T10:00:00Z", "main", 0)).has_value());
|
||||
CHECK(tasks.insert(task("uc1", "queued", "2026-09-10T10:00:01Z", "main", 1)).has_value());
|
||||
CHECK(sched.tick().has_value());
|
||||
CHECK_EQ(engine.starts.size(), 2u);
|
||||
|
||||
auto c = sched.user_cancel("uc0", /*discard_partial=*/true);
|
||||
CHECK(c.found);
|
||||
CHECK(c.changed);
|
||||
CHECK_EQ(c.state, std::string("cancelled"));
|
||||
CHECK_EQ(engine.cancelled.size(), 1u);
|
||||
CHECK(engine.cancelled[0].second); // discard_partial passed through
|
||||
CHECK_EQ(task_state(*db, "uc0"), std::string("cancelled"));
|
||||
|
||||
// Cancelling an already-terminal task is a no-op.
|
||||
auto c2 = sched.user_cancel("uc0", false);
|
||||
CHECK(c2.found);
|
||||
CHECK(!c2.changed);
|
||||
|
||||
// pause_queue pauses every still-running task in the queue (uc0 is terminal, so
|
||||
// only uc1 is affected) and reports pause_reason 'queue_stopped'.
|
||||
const auto paused_ids = sched.pause_queue("main");
|
||||
CHECK_EQ(paused_ids.size(), std::size_t{1});
|
||||
CHECK_EQ(paused_ids[0], std::string("uc1"));
|
||||
CHECK_EQ(task_state(*db, "uc1"), std::string("paused"));
|
||||
auto row = tasks.get("uc1").value().value();
|
||||
CHECK_EQ(row.pause_reason.value_or(""), std::string("queue_stopped"));
|
||||
|
||||
tasks.remove("uc0");
|
||||
tasks.remove("uc1");
|
||||
CHECK(db->exec("UPDATE queues SET state='stopped' WHERE queue_id='main'").has_value());
|
||||
}
|
||||
|
||||
// --- probe_now: success (with a category guess) and a mapped failure ------------
|
||||
{
|
||||
FakeEnginePort engine;
|
||||
Scheduler sched(*db, engine,
|
||||
Governor(GovernorConfig{.max_concurrent_downloads = 10,
|
||||
.max_active_segments = 32}));
|
||||
|
||||
vdm::net::ProbeResult pr;
|
||||
pr.effective_url = "https://cdn.example/movie.mp4";
|
||||
pr.filename_from_url = "movie.mp4"; // suggest_filename() needs this set; the real
|
||||
// Prober fills it from the URL path itself
|
||||
pr.total_size = 123456;
|
||||
pr.mime = "video/mp4";
|
||||
pr.resumable = true;
|
||||
pr.accept_ranges = true;
|
||||
pr.etag = "\"abc\"";
|
||||
engine.auto_probe_result = vdm::Result<vdm::net::ProbeResult>{pr};
|
||||
|
||||
velox::proto::DownloadProbeParams params;
|
||||
params.url = "https://cdn.example/movie.mp4";
|
||||
std::optional<velox::proto::HandlerResult<velox::proto::DownloadProbeResult>> got;
|
||||
sched.probe_now(params, [&](auto r) { got = std::move(r); });
|
||||
CHECK(got.has_value());
|
||||
CHECK(got->has_value());
|
||||
if (got && *got) {
|
||||
const auto& r = **got;
|
||||
CHECK_EQ(r.sizeBytes.value_or(-1), std::int64_t{123456});
|
||||
CHECK(r.resumable);
|
||||
CHECK_EQ(r.mime, std::string("video/mp4"));
|
||||
// movie.mp4 -> the 'video' built-in category by extension.
|
||||
CHECK_EQ(r.suggestedCategoryId, std::string("video"));
|
||||
}
|
||||
|
||||
vdm::ErrorInfo err;
|
||||
err.code = vdm::Error::connect_failed;
|
||||
err.context = "connection refused";
|
||||
engine.auto_probe_result = vdm::Result<vdm::net::ProbeResult>{err};
|
||||
std::optional<velox::proto::HandlerResult<velox::proto::DownloadProbeResult>> got2;
|
||||
sched.probe_now(params, [&](auto r) { got2 = std::move(r); });
|
||||
CHECK(got2.has_value());
|
||||
CHECK(!got2->has_value());
|
||||
if (got2 && !*got2)
|
||||
CHECK(got2->error().code == velox::proto::ErrorCode::ProbeFailed);
|
||||
}
|
||||
|
||||
// --- refresh_url: content unchanged, content changed, and a live-handle swap -----
|
||||
{
|
||||
FakeEnginePort engine;
|
||||
Scheduler sched(*db, engine,
|
||||
Governor(GovernorConfig{.max_concurrent_downloads = 10,
|
||||
.max_active_segments = 32}));
|
||||
store::TaskRow r;
|
||||
r.task_id = "ru0";
|
||||
r.url = "https://cdn.example/old-signed-url";
|
||||
r.save_dir = "/tmp";
|
||||
r.filename = "ru0.bin";
|
||||
r.state = "queued";
|
||||
r.created_at = "2026-09-12T00:00:00Z";
|
||||
r.size_bytes = 1000;
|
||||
r.etag = "\"same\"";
|
||||
CHECK(tasks.insert(r).has_value());
|
||||
CHECK(sched.tick().has_value());
|
||||
CHECK_EQ(engine.starts.size(), 1u);
|
||||
const auto live_id = engine.starts[0].id;
|
||||
|
||||
// Same size and etag: not changed.
|
||||
vdm::net::ProbeResult pr;
|
||||
pr.effective_url = "https://cdn.example/new-signed-url";
|
||||
pr.total_size = 1000;
|
||||
pr.etag = "\"same\"";
|
||||
pr.resumable = true;
|
||||
engine.auto_probe_result = vdm::Result<vdm::net::ProbeResult>{pr};
|
||||
|
||||
std::optional<velox::proto::HandlerResult<velox::proto::DownloadRefreshUrlResult>> got;
|
||||
sched.refresh_url("ru0", "https://cdn.example/new-signed-url", std::nullopt, std::nullopt,
|
||||
[&](auto r2) { got = std::move(r2); });
|
||||
CHECK(got.has_value());
|
||||
CHECK(got->has_value());
|
||||
if (got && *got) {
|
||||
CHECK((*got)->ok);
|
||||
CHECK(!(*got)->contentChanged);
|
||||
CHECK((*got)->resumable);
|
||||
}
|
||||
// The live handle got its URL swapped, not a fresh start().
|
||||
CHECK_EQ(engine.starts.size(), 1u);
|
||||
CHECK_EQ(engine.refreshed_urls.size(), std::size_t{1});
|
||||
if (!engine.refreshed_urls.empty()) {
|
||||
CHECK_EQ(engine.refreshed_urls[0].id.value, live_id.value);
|
||||
CHECK_EQ(engine.refreshed_urls[0].url, std::string("https://cdn.example/new-signed-url"));
|
||||
}
|
||||
auto row = tasks.get("ru0").value().value();
|
||||
CHECK_EQ(row.url, std::string("https://cdn.example/new-signed-url"));
|
||||
|
||||
// A different size: content changed.
|
||||
pr.total_size = 2000;
|
||||
engine.auto_probe_result = vdm::Result<vdm::net::ProbeResult>{pr};
|
||||
std::optional<velox::proto::HandlerResult<velox::proto::DownloadRefreshUrlResult>> got2;
|
||||
sched.refresh_url("ru0", "https://cdn.example/another-url", std::nullopt, std::nullopt,
|
||||
[&](auto r2) { got2 = std::move(r2); });
|
||||
CHECK(got2.has_value() && got2->has_value());
|
||||
if (got2 && *got2) CHECK((*got2)->contentChanged);
|
||||
|
||||
// Unknown task: TaskNotFound, no probe issued.
|
||||
const auto probes_before = engine.probe_requests.size();
|
||||
std::optional<velox::proto::HandlerResult<velox::proto::DownloadRefreshUrlResult>> got3;
|
||||
sched.refresh_url("does-not-exist", "https://x/y", std::nullopt, std::nullopt,
|
||||
[&](auto r2) { got3 = std::move(r2); });
|
||||
CHECK(got3.has_value() && !got3->has_value());
|
||||
if (got3 && !*got3) CHECK(got3->error().code == velox::proto::ErrorCode::TaskNotFound);
|
||||
CHECK_EQ(engine.probe_requests.size(), probes_before);
|
||||
|
||||
tasks.remove("ru0");
|
||||
}
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
// The single-instance lock: isolated by runtime dir, not just by euid. This is the bug a
|
||||
// leaked test veloxd exploited — one abstract-socket name per user meant every isolated
|
||||
// instance (real daemon, tests, other lanes) fought over the same lock.
|
||||
|
||||
#include <unistd.h>
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/single_instance.hpp"
|
||||
|
||||
using namespace velox::daemon::rpc;
|
||||
|
||||
void run() {
|
||||
// Two different runtime dirs: both acquire the lock independently.
|
||||
{
|
||||
const int a = acquire_single_instance_lock("/run/user/1000/velox-test-a");
|
||||
const int b = acquire_single_instance_lock("/run/user/1000/velox-test-b");
|
||||
CHECK(a >= 0);
|
||||
CHECK(b >= 0);
|
||||
if (a >= 0) ::close(a);
|
||||
if (b >= 0) ::close(b);
|
||||
}
|
||||
|
||||
// Same runtime dir: the second attempt is refused while the first still holds it.
|
||||
{
|
||||
const int first = acquire_single_instance_lock("/run/user/1000/velox-test-shared");
|
||||
CHECK(first >= 0);
|
||||
const int second = acquire_single_instance_lock("/run/user/1000/velox-test-shared");
|
||||
CHECK(second < 0);
|
||||
if (first >= 0) ::close(first);
|
||||
if (second >= 0) ::close(second);
|
||||
|
||||
// Releasing (closing) the fd frees the abstract-namespace name immediately — a
|
||||
// third attempt at the same dir succeeds once the first is gone.
|
||||
const int third = acquire_single_instance_lock("/run/user/1000/velox-test-shared");
|
||||
CHECK(third >= 0);
|
||||
if (third >= 0) ::close(third);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
@@ -0,0 +1,179 @@
|
||||
// store/categories + store/queues: the two D3 handlers GUI's category panel and queue
|
||||
// view need against a real daemon.
|
||||
|
||||
#include <algorithm>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
#include "check.hpp"
|
||||
#include "store/categories.hpp"
|
||||
#include "store/migrations.hpp"
|
||||
#include "store/queues.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "store/tasks.hpp"
|
||||
|
||||
using namespace velox::daemon::store;
|
||||
|
||||
void run() {
|
||||
auto db = Db::open(":memory:");
|
||||
CHECK(db.has_value());
|
||||
if (!db) return;
|
||||
CHECK(migrate_to_head(*db).has_value());
|
||||
|
||||
// --- categories: the six seeded built-ins, builtin first --------------------
|
||||
{
|
||||
Categories categories(*db);
|
||||
auto items = categories.list();
|
||||
CHECK(items.has_value());
|
||||
if (items) {
|
||||
CHECK_EQ(items->size(), 6u);
|
||||
for (const auto& c : *items) CHECK(c.builtin);
|
||||
const auto& programs =
|
||||
*std::find_if(items->begin(), items->end(),
|
||||
[](const auto& c) { return c.categoryId == "programs"; });
|
||||
CHECK(!programs.extensions.empty());
|
||||
CHECK(std::find(programs.extensions.begin(), programs.extensions.end(), "iso") !=
|
||||
programs.extensions.end());
|
||||
}
|
||||
}
|
||||
|
||||
// --- queues: the seeded "main" queue, empty task list ------------------------
|
||||
{
|
||||
Queues queues(*db);
|
||||
auto items = queues.list();
|
||||
CHECK(items.has_value());
|
||||
if (items) {
|
||||
CHECK_EQ(items->size(), 1u);
|
||||
CHECK_EQ(items->front().queueId, std::string("main"));
|
||||
CHECK(items->front().taskIds.has_value());
|
||||
CHECK(items->front().taskIds->empty());
|
||||
}
|
||||
}
|
||||
|
||||
// --- queue.taskIds reflects membership, in queue_position order ------------
|
||||
{
|
||||
Tasks tasks(*db);
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
TaskRow r;
|
||||
r.task_id = "t" + std::to_string(i);
|
||||
r.url = "https://example.com/" + r.task_id;
|
||||
r.save_dir = "/tmp";
|
||||
r.filename = r.task_id;
|
||||
r.created_at = "2026-09-11T00:00:00Z";
|
||||
r.queue_id = "main";
|
||||
r.queue_position = 2 - i; // reverse insertion order
|
||||
CHECK(tasks.insert(r).has_value());
|
||||
}
|
||||
Queues queues(*db);
|
||||
auto items = queues.list();
|
||||
CHECK(items.has_value());
|
||||
if (items && !items->empty()) {
|
||||
const auto& ids = *items->front().taskIds;
|
||||
CHECK_EQ(ids.size(), 3u);
|
||||
CHECK_EQ(ids[0], std::string("t2")); // queue_position 0
|
||||
CHECK_EQ(ids[1], std::string("t1"));
|
||||
CHECK_EQ(ids[2], std::string("t0"));
|
||||
}
|
||||
}
|
||||
|
||||
// --- Categories::upsert: create generates an id, builtin is never settable ------
|
||||
{
|
||||
Categories categories(*db);
|
||||
|
||||
velox::proto::Category in;
|
||||
in.name = "ISOs";
|
||||
in.saveDir = "/tmp/isos";
|
||||
in.extensions = {"iso"};
|
||||
in.builtin = true; // ignored on create: a client cannot mint a builtin category
|
||||
auto created = categories.upsert(in);
|
||||
CHECK(created.has_value());
|
||||
if (created) {
|
||||
CHECK(!created->categoryId.empty());
|
||||
CHECK(!created->builtin);
|
||||
|
||||
// A replace keeps builtin=false too, and can rename/re-point.
|
||||
velox::proto::Category patch = *created;
|
||||
patch.name = "ISO Images";
|
||||
patch.builtin = true; // still ignored
|
||||
auto replaced = categories.upsert(patch);
|
||||
CHECK(replaced.has_value());
|
||||
if (replaced) {
|
||||
CHECK_EQ(replaced->categoryId, created->categoryId);
|
||||
CHECK_EQ(replaced->name, std::string("ISO Images"));
|
||||
CHECK(!replaced->builtin);
|
||||
}
|
||||
|
||||
// A builtin category is untouched by remove(), and its tasks are not
|
||||
// reassigned away from it — the store enforces this even without the
|
||||
// dispatcher's own -32602 pre-check.
|
||||
auto builtin_attempt = categories.remove("general", std::nullopt);
|
||||
CHECK(builtin_attempt.has_value());
|
||||
if (builtin_attempt) CHECK(!builtin_attempt->removed);
|
||||
|
||||
// remove() reassigns member tasks (default target: "general") and deletes
|
||||
// the row.
|
||||
Tasks tasks(*db);
|
||||
TaskRow r;
|
||||
r.task_id = "cat-owner";
|
||||
r.url = "https://example.com/x";
|
||||
r.save_dir = "/tmp";
|
||||
r.filename = "x";
|
||||
r.created_at = "2026-09-11T00:00:00Z";
|
||||
r.category_id = created->categoryId;
|
||||
CHECK(tasks.insert(r).has_value());
|
||||
|
||||
auto removed = categories.remove(created->categoryId, std::nullopt);
|
||||
CHECK(removed.has_value());
|
||||
if (removed) {
|
||||
CHECK(removed->removed);
|
||||
CHECK_EQ(removed->reassigned_task_ids.size(), std::size_t{1});
|
||||
CHECK_EQ(removed->reassigned_task_ids[0], std::string("cat-owner"));
|
||||
}
|
||||
auto owner = tasks.get("cat-owner");
|
||||
CHECK(owner.has_value() && owner->has_value());
|
||||
if (owner && *owner)
|
||||
CHECK_EQ((*owner)->category_id.value_or(""), std::string("general"));
|
||||
|
||||
// Gone: a second remove() finds nothing.
|
||||
auto gone = categories.remove(created->categoryId, std::nullopt);
|
||||
CHECK(gone.has_value());
|
||||
if (gone) CHECK(!gone->removed);
|
||||
|
||||
tasks.remove("cat-owner");
|
||||
}
|
||||
}
|
||||
|
||||
// --- Queues::upsert: create generates an id; replace keeps the run state --------
|
||||
{
|
||||
Queues queues(*db);
|
||||
|
||||
velox::proto::Queue in;
|
||||
in.name = "Nightly";
|
||||
in.state = velox::proto::QueueState::Running; // ignored on create: always 'stopped'
|
||||
in.maxConcurrent = 3;
|
||||
auto created = queues.upsert(in);
|
||||
CHECK(created.has_value());
|
||||
if (created) {
|
||||
CHECK(!created->queueId.empty());
|
||||
CHECK(created->state == velox::proto::QueueState::Stopped);
|
||||
|
||||
CHECK(queues.set_state(created->queueId, "running").has_value());
|
||||
|
||||
velox::proto::Queue patch = *created;
|
||||
patch.name = "Nightly Batch";
|
||||
patch.maxConcurrent = 5;
|
||||
patch.state = velox::proto::QueueState::Stopped; // ignored on replace too
|
||||
auto replaced = queues.upsert(patch);
|
||||
CHECK(replaced.has_value());
|
||||
if (replaced) {
|
||||
CHECK_EQ(replaced->name, std::string("Nightly Batch"));
|
||||
CHECK_EQ(replaced->maxConcurrent, std::int64_t{5});
|
||||
// Run state survived the config edit — still 'running' from set_state above,
|
||||
// not reset by the payload's (ignored) 'stopped'.
|
||||
CHECK(replaced->state == velox::proto::QueueState::Running);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
@@ -86,6 +86,56 @@ void run() {
|
||||
.has_value());
|
||||
}
|
||||
|
||||
// --- 0003: start_mode is rebuilt to the contract's values, existing rows mapped -
|
||||
{
|
||||
auto db = Db::open(":memory:");
|
||||
CHECK(db.has_value());
|
||||
if (!db) return;
|
||||
// Build a real pre-0003 db (schema 1..2) with rows in the old, non-contract
|
||||
// start_mode spelling, the way an actually-released daemon would have them.
|
||||
for (const auto& m : embedded_migrations()) {
|
||||
if (m.version > 2) break;
|
||||
CHECK(db->exec(m.sql).has_value());
|
||||
CHECK(db->set_user_version(m.version).has_value());
|
||||
}
|
||||
CHECK(db->exec("INSERT INTO tasks(task_id,url,save_dir,created_at,start_mode) "
|
||||
"VALUES('auto1','http://x','/tmp','2026-09-10T00:00:00Z','auto')")
|
||||
.has_value());
|
||||
CHECK(db->exec("INSERT INTO tasks(task_id,url,save_dir,created_at,start_mode) "
|
||||
"VALUES('man1','http://x','/tmp','2026-09-10T00:00:00Z','manual')")
|
||||
.has_value());
|
||||
CHECK(db->exec("INSERT INTO tasks(task_id,url,save_dir,created_at,start_mode) "
|
||||
"VALUES('q1','http://x','/tmp','2026-09-10T00:00:00Z','queue')")
|
||||
.has_value());
|
||||
|
||||
CHECK(migrate_to_head(*db).has_value());
|
||||
CHECK_EQ(db->user_version(), head);
|
||||
|
||||
auto start_mode_of = [&](const char* id) -> std::string {
|
||||
auto st = db->prepare("SELECT start_mode FROM tasks WHERE task_id=?1");
|
||||
if (!st || !st->bind(1, std::string_view(id))) return "";
|
||||
auto row = st->step();
|
||||
if (!row || !*row) return "";
|
||||
return std::string(st->column_text(0));
|
||||
};
|
||||
CHECK_EQ(start_mode_of("auto1"), std::string("now"));
|
||||
CHECK_EQ(start_mode_of("man1"), std::string("later"));
|
||||
CHECK_EQ(start_mode_of("q1"), std::string("queue")); // passes through unchanged
|
||||
|
||||
// The bug this migration closes: 'later' — a real, documented StartMode value —
|
||||
// used to hit the old CHECK and fail every insert. It's accepted now, and the two
|
||||
// retired spellings are gone for good.
|
||||
CHECK(db->exec("INSERT INTO tasks(task_id,url,save_dir,created_at,start_mode) "
|
||||
"VALUES('later1','http://x','/tmp','2026-09-10T00:00:00Z','later')")
|
||||
.has_value());
|
||||
CHECK(db->exec("INSERT INTO tasks(task_id,url,save_dir,created_at,start_mode) "
|
||||
"VALUES('bad1','http://x','/tmp','2026-09-10T00:00:00Z','auto')")
|
||||
.has_value() == false);
|
||||
CHECK(db->exec("INSERT INTO tasks(task_id,url,save_dir,created_at,start_mode) "
|
||||
"VALUES('bad2','http://x','/tmp','2026-09-10T00:00:00Z','manual')")
|
||||
.has_value() == false);
|
||||
}
|
||||
|
||||
// --- re-running the migrator on an at-head DB is a no-op ------------------------
|
||||
{
|
||||
auto db = Db::open(":memory:");
|
||||
@@ -100,11 +150,20 @@ void run() {
|
||||
}
|
||||
|
||||
// --- forward-only: from every released version [0 .. head-1], reach head --------
|
||||
// A "released version N" db has the real schema migrations 1..N actually built, not
|
||||
// just the pragma set to N — faking the pragma alone left `start >= 1` cases running
|
||||
// a later migration (e.g. 0002's ALTER TABLE tasks / rebuild of segments) against a
|
||||
// db with no tables at all.
|
||||
for (std::int64_t start = 0; start < head; ++start) {
|
||||
auto db = Db::open(":memory:");
|
||||
CHECK(db.has_value());
|
||||
if (!db) continue;
|
||||
CHECK(db->set_user_version(start).has_value());
|
||||
for (const auto& m : embedded_migrations()) {
|
||||
if (m.version > start) break;
|
||||
CHECK(db->exec(m.sql).has_value());
|
||||
CHECK(db->set_user_version(m.version).has_value());
|
||||
}
|
||||
CHECK_EQ(db->user_version(), start);
|
||||
auto out = migrate_to_head(*db);
|
||||
CHECK(out.has_value());
|
||||
if (out) {
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/event_loop.hpp"
|
||||
#include "rpc/ndjson.hpp"
|
||||
#include "rpc/uds_server.hpp"
|
||||
@@ -71,8 +72,9 @@ void run() {
|
||||
CHECK(velox::daemon::store::migrate_to_head(*db).has_value());
|
||||
|
||||
rpc::EventLoop loop;
|
||||
rpc::VeloxDispatcher dispatcher(*db);
|
||||
rpc::UdsServer server(loop, dispatcher, sock);
|
||||
rpc::EventHub hub;
|
||||
rpc::VeloxDispatcher dispatcher(*db, hub);
|
||||
rpc::UdsServer server(loop, dispatcher, hub, sock);
|
||||
const auto ec = server.start();
|
||||
CHECK(!ec);
|
||||
if (ec) return;
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/event_loop.hpp"
|
||||
#include "rpc/pairing.hpp"
|
||||
#include "rpc/runtime_dir.hpp"
|
||||
@@ -123,9 +124,10 @@ void run() {
|
||||
rpc::RuntimeDir rt{dir ? dir : "/tmp"};
|
||||
|
||||
rpc::EventLoop loop;
|
||||
rpc::VeloxDispatcher dispatcher(*db);
|
||||
rpc::EventHub hub;
|
||||
rpc::VeloxDispatcher dispatcher(*db, hub);
|
||||
rpc::EnvAutoApprover approver;
|
||||
rpc::WsServer server(loop, dispatcher, *db, approver, rt);
|
||||
rpc::WsServer server(loop, dispatcher, *db, approver, hub, rt);
|
||||
const auto ec = server.start();
|
||||
CHECK(!ec);
|
||||
if (ec) return;
|
||||
|
||||
@@ -162,4 +162,34 @@ extension/
|
||||
|
||||
`<all_urls>` is unavoidable for a download manager but is the main review-friction item:
|
||||
document *why* in the AMO submission notes and in the source, and make the exclusion list
|
||||
prominent in Options.
|
||||
prominent in Options. The submission-ready copy of this table lives in
|
||||
`extension/docs/amo-permissions.md`, committed alongside the manifest it explains.
|
||||
|
||||
## 8. Popup and Options talk to the background page over a relay, not directly
|
||||
|
||||
`popup/` and `options/` are separate documents (a browser action popup and an
|
||||
`options_ui` page) — they cannot import `background/index.ts`'s live `VeloxTransport`.
|
||||
`background/bridge.ts` relays it over one `browser.runtime.connect` port per document:
|
||||
`call`/`subscribe`/`getStatus`/`reconnect`/`pair`/`unpair`/`setOverride` requests in,
|
||||
`result`/`event`/`status`/`pairError` responses out. `shared/panel-client.ts` is the
|
||||
client side both surfaces use. The status payload carries `kind` (which transport
|
||||
implementation is live) alongside the existing `TransportStatus`, because Options needs
|
||||
it: `settings.set` and `rules.upsert` are privileged, uds-only methods (see
|
||||
`shared/protocol/methods.ts`'s `METHODS` table), so the capture-policy edit form only
|
||||
ever unlocks when the active transport is native messaging. Over WebSocket — the
|
||||
default, guaranteed path per ADR 0003 — Options renders the daemon's policy read-only
|
||||
via `capture.getRules` rather than pretending it can write settings the protocol
|
||||
refuses over that transport. The one Options setting that genuinely belongs to the
|
||||
extension (not the daemon) — the default category applied to extension-initiated
|
||||
downloads — lives in `browser.storage.local` via `options/prefs.ts`, the same place the
|
||||
pairing token and transport override already live.
|
||||
|
||||
Streaming-media detection (build step 7) also splits down this line: `capture/media.ts`
|
||||
(background, webRequest-based) recognizes `.m3u8`/`.mpd` URLs and their content types
|
||||
and tells the tab's content script over `runtime.sendMessage`
|
||||
(`background/media-bridge.ts`); `content/media-observer.ts` independently watches the
|
||||
page's own `<video>` elements for the same signal. Either one showing up opens
|
||||
`content/video-panel.ts`'s "Download this video ▾" panel, which calls
|
||||
`media.listVariants`/`media.addVariant` through the background page and greys out any
|
||||
variant (or the whole manifest) flagged `drm`/`drmProtected` with "Protected content" —
|
||||
the extension never parses the manifest itself.
|
||||
|
||||
@@ -53,6 +53,24 @@ capture, and the Add-URL dialog pre-fills from the clipboard when opened. Backgr
|
||||
monitoring is a bonus if the spike says yes. **Do not let this block the release, and do
|
||||
not promise it in the UI before S2 answers.**
|
||||
|
||||
**Spike S2, item (c) answered — verified live, not the full spike:** on this desktop
|
||||
(GNOME/Mutter, Ubuntu 26.04, plain non-Flatpak/non-snap process), `CreateSession` on
|
||||
`org.freedesktop.portal.GlobalShortcuts` refuses every caller with `"An app id is
|
||||
required"` — reproduced two ways: `gui/src/clipboard/GlobalShortcut.cpp`'s real async
|
||||
D-Bus call, and a bare `busctl --user call … CreateSession` from an interactive shell
|
||||
(no Qt involved at all), both under a real Wayland session (`WAYLAND_DISPLAY` set), not
|
||||
just offscreen. Because the second reproduction has no Qt/app-level identity to configure
|
||||
at all and still fails identically, this looks like the portal requiring the caller's
|
||||
*bus connection* to already carry a sandboxed app id (Flatpak/snap portal-confined) —
|
||||
something no amount of `QGuiApplication::setDesktopFileName()` or similar can supply from
|
||||
an unconfined process. **Practical read:** the global-shortcut explicit path likely does
|
||||
not work at all for Velox as a traditionally-packaged (.deb/AppImage) app on stock
|
||||
GNOME — only if/when it ships confined. The code is still in (best-effort, fails silent
|
||||
exactly like this, never advertised — see the file's own header), since it costs nothing
|
||||
and activates automatically the day that changes. Items (a) `QClipboard::dataChanged`
|
||||
cross-app, (b) `wlr-data-control`, and (d) XWayland fallback are **still unanswered** —
|
||||
this was one item of S2's four, not the full spike.
|
||||
|
||||
---
|
||||
|
||||
## R3 — AMO review friction 🟠 MEDIUM
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
# 16. Global rate limit fairness under heavy segment contention (known issue, not fixed)
|
||||
|
||||
Status: accepted (documents a known limitation; no code change to `rate::RateLimiter`)
|
||||
|
||||
## Context
|
||||
|
||||
While finishing `tools/bench`'s `load` subcommand (the M1 DoD's 20-task load test), pacing
|
||||
20 concurrent tasks (`default_segments=8` each, so up to 160 segments contending for
|
||||
`max_active_segments=32` slots) via `RateLimiter::set_global_limit()` reproduced 2 of 20
|
||||
tasks hanging past a 120 s per-task wait instead of completing in the ~5 s the configured
|
||||
rate implied. Unthrottled, all 20 tasks complete in well under a second — the stall is
|
||||
specific to many segment workers contending for one global `TokenBucket` through
|
||||
`RateLimiter::acquire()`'s peek-then-commit-all-or-nothing path, not a general deadlock.
|
||||
|
||||
`TokenBucket::consume`/`peek` compute a wait duration assuming the caller retries once that
|
||||
much time has passed and the bucket will then have `n` tokens. Under heavy contention that
|
||||
assumption breaks: many segment workers independently schedule a retry via
|
||||
`TaskHost::schedule()` for whenever they were told tokens *would* be available, but
|
||||
whichever of them acquires `RateLimiter::mu_` first on waking drains the tokens the others
|
||||
were counting on, forcing the losers to recompute and reschedule a fresh wait. There is no
|
||||
fairness ordering (FIFO queue, ticket, or similar) across that race — repeated bad luck for
|
||||
the same task's segments is possible and was observed twice in one run. This gets worse,
|
||||
not better, as contention rises: more competing waiters means more of them lose each round.
|
||||
|
||||
## Decision
|
||||
|
||||
Left unfixed for M1. `tools/bench/vdm_bench.cpp`'s `load` subcommand works around it by
|
||||
giving each task its own independent per-task bucket (`RateLimiter::set_task_limit`)
|
||||
instead of one shared global bucket — each task's `acquire()` then only ever contends with
|
||||
its own ≤8 segments, which the same 20-task run completes in ~6.6 s with zero timeouts. That
|
||||
workaround is sufficient for the bench (it still needs, and gets, real concurrent buffering
|
||||
to measure RSS against) and is documented inline where the choice is made.
|
||||
|
||||
It is **not** sufficient for a real user: `download.setGlobalLimit` (or however DAEMON
|
||||
surfaces it) is a real, everyday feature, and a household running 20+ concurrent downloads
|
||||
against a single global cap is a plausible, not exotic, scenario. This ADR exists so that
|
||||
scenario doesn't get rediscovered from scratch.
|
||||
|
||||
## Consequences
|
||||
|
||||
- `RateLimiter`'s global/queue level should get a fairness mechanism before M1 sign-off
|
||||
treats "global bandwidth cap with many concurrent downloads" as supported: e.g. serve
|
||||
waiters in the order their wait was computed (a min-heap keyed on wake time, or a simple
|
||||
ticket counter checked before committing), or move to a scheme where a waiting caller's
|
||||
reserved allocation can't be stolen by a later arrival.
|
||||
- Needs a regression test once fixed: N tasks (N large enough to exceed
|
||||
`max_active_segments`), one shared global limit, assert every task completes within a
|
||||
bounded multiple of the ideal `total_bytes / global_bps` time — the shape of the bug
|
||||
`tools/bench load` stumbled into, made deterministic.
|
||||
- Filed here rather than fixed in this change because it's a `core/src/rate/` /
|
||||
`core/src/task/` design question (retry/backoff and scheduling policy under contention),
|
||||
not a `tools/bench` one, and deserves its own review rather than a bundled-in fix.
|
||||
|
||||
## Postscript: a second, TSan-only straggler (still unexplained, not proven the same bug)
|
||||
|
||||
After the `--preset tsan` build was made to work (a real, separate ASan-caught bug fixed in
|
||||
the same change: `DownloadTaskState::quiesce()` was clearing `workers` synchronously right
|
||||
after issuing an async `cancel()`, racing the HttpClient worker thread's still-in-flight
|
||||
write callback — see the commit that adds this ADR), `tools/bench load` was run under
|
||||
`--preset tsan` to complete the M1 DoD's sanitizer-clean load test. Even with the
|
||||
per-task-bucket workaround above *and* external (testserver-side) pacing instead of the
|
||||
engine's rate limiter entirely, a single straggler task failed to complete within a
|
||||
generous (300–500s) per-task budget under TSan specifically — reproduced at
|
||||
tasks=20/segments=8 (2 stragglers), tasks=20/segments=2 (1 straggler); tasks=8/segments=2
|
||||
was reliable across repeated runs and is what `tools/bench`'s ctest registration now uses.
|
||||
|
||||
No TSan diagnostic (data race, lock-order inversion, etc.) ever accompanied a straggler —
|
||||
across every run that hit one. That means either: (a) it really is just TSan's per-access
|
||||
instrumentation overhead compounding with this sandbox's own scheduling/virtualization
|
||||
under high simultaneous curl/thread activity, with no engine defect at all, or (b) it's a
|
||||
genuine timing-sensitive bug (a plausible candidate: `HttpClient`'s
|
||||
`CURLOPT_LOW_SPEED_LIMIT`/`CURLOPT_LOW_SPEED_TIME` stall detection — 1024 B/s for 30s by
|
||||
default — false-tripping when TSan's overhead makes real throughput look stalled to curl's
|
||||
own timers, driving a segment into a retry/backoff loop that never catches up) that TSan's
|
||||
slowdown merely makes likelier to manifest, not one it creates. This was not root-caused:
|
||||
doing so needs reproducing outside this sandbox, on hardware not already shared/loaded, to
|
||||
separate "TSan is just slow here" from "there is a real bug TSan is making easier to hit".
|
||||
|
||||
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
|
||||
@@ -0,0 +1,102 @@
|
||||
# 17. SegmentBudget over-admission, and a wait-list to wake denied tasks
|
||||
|
||||
Status: accepted
|
||||
|
||||
## Context
|
||||
|
||||
`core/docs/m7-baseline.md`'s first pass measured ~70 MB RSS for the M1/M7 DoD's "20 active
|
||||
downloads, `max_active_segments=32`" scenario, against a 60 MB target `docs/adr/0012`
|
||||
estimated would hold with ~45–50 MB of margin. `tools/bench heap-profile` (added to chase
|
||||
this down — no massif/heaptrack in this environment) found the actual cause directly:
|
||||
`Engine::segment_budget().budget().active` read 56–86 against a `total` of 32. The engine
|
||||
was not over budget by a measurement artifact or an ADR 0012 arithmetic gap; it was
|
||||
genuinely running more concurrent segments — and their 1 MiB ring buffers — than
|
||||
`max_active_segments` allows.
|
||||
|
||||
`SegmentBudget::confirm_slot(id)` checked only `t.held < t.target` — a per-task check.
|
||||
`reallocate_locked()`'s two-pass fairness allocation does correctly bound
|
||||
`sum(target) <= max_active_` at the moment it computes a plan, but that bound says nothing
|
||||
about `sum(held)`: a task can be legitimately holding more than its own just-lowered
|
||||
target for a while, because yield is deliberately deferred to a segment boundary, never
|
||||
mid-segment (ADR 0011 A1). When that happens, another task's target can correctly rise to
|
||||
claim the capacity the first task is *about* to give back, and both `confirm_slot()` calls
|
||||
can succeed against their own, individually-correct targets while the sum of what's
|
||||
*physically held* exceeds `max_active_`. Reproduced deterministically in
|
||||
`core/tests/segment/budget_test.cpp` (`budget_active_never_exceeds_max_active_segments_
|
||||
under_concurrent_load`, `budget_wait_list_wakes_a_task_whose_target_never_changed`)
|
||||
independent of any real network I/O.
|
||||
|
||||
## Decision
|
||||
|
||||
**`confirm_slot()` also checks `active_ < max_active_`, unconditionally**, as a backstop
|
||||
that doesn't depend on any task's target bookkeeping being perfectly in sync with what
|
||||
every other task currently holds. This is the fix for the invariant itself.
|
||||
|
||||
That backstop creates a liveness question the original design never had to answer: a task
|
||||
denied *only* by this new check has a target that's already correct — it doesn't change
|
||||
again on its own, so `reallocate_locked()`'s plain "fire a callback when a task's target
|
||||
changes" mechanism never revisits it. Nothing was watching for "this specific task is owed
|
||||
a slot"; **`Task` gained a `waiting_for_slot` flag**, set by `confirm_slot()` on exactly
|
||||
this denial and cleared on the task's next successful confirm however that retry was
|
||||
triggered. `release_slot()` and `deregister_task()` — the only two places that can free
|
||||
real capacity — now call `wake_one_waiter_locked()`, which hands the newly-freed slot to
|
||||
the highest-priority `waiting_for_slot` task (other than the one that just released) via a
|
||||
direct retry hint, if `reallocate_locked()`'s own plan didn't already produce one.
|
||||
|
||||
On the `download_task.cpp` side, a woken task's `apply_slot_target()` → `fill_slots_locked()`
|
||||
also needed a fix: a segment that had backed off mid-retry (`SegState::stalled`, its
|
||||
`release_slot()` already called, a `retry_worker()` timer already scheduled and possibly
|
||||
already denied once) has no live worker and never surfaces through
|
||||
`Segmenter::assign_slot()` — it stays *assigned*, just not running, and assign_slot() only
|
||||
ever hands out unassigned or fresh ranges. `fill_slots_locked()` now restarts any stalled
|
||||
segment with no live worker directly (bounded by `slot_target`, same as its `assign_slot()`
|
||||
loop) before looking for new work; a segment it doesn't get to keeps its own scheduled
|
||||
`retry_worker()` timer as a second chance, so this is additive, not a replacement for that
|
||||
path.
|
||||
|
||||
### What this isn't
|
||||
|
||||
Production's real `on_target` callback (`register_task()`'s lambda, `download_task.cpp`)
|
||||
never runs synchronously with whatever budget call triggered it — it posts through
|
||||
`host.schedule()` (`engine.cpp`'s single timer thread), so a task's own `mu` can never be
|
||||
re-entered on the same call stack, and two tasks' callbacks can never race each other into
|
||||
an AB-BA lock order either (only one ever runs at a time, on one thread). An earlier
|
||||
version of this fix tried to defend against a same-thread reentrancy hazard that,
|
||||
diagnosed correctly, doesn't exist in production at all — it exists only if a *test double*
|
||||
calls back into the budget synchronously from inside `on_target`, which none of production
|
||||
does. `core/tests/segment/budget_test.cpp`'s `FakeTask` does exactly that (by design — it's
|
||||
simple and every other test in the file drives the budget from one thread at a time, where
|
||||
that's harmless); the one test that drives it from *multiple* concurrent threads
|
||||
(`budget_concurrent_confirm_release_stays_consistent`) uses a separate `AsyncFakeTask` that
|
||||
posts through a small `TestTimer`, mirroring `host.schedule()` / the engine's timer thread
|
||||
for real, rather than adding synchronization machinery to `SegmentBudget` itself to paper
|
||||
over a test double being unlike production. `wake_one_waiter_locked()`'s `exclude`
|
||||
parameter is the one piece of that defense that's independently justified either way — a
|
||||
task doesn't need to be told to retry a slot it just gave back itself — and is the only
|
||||
piece that stayed.
|
||||
|
||||
## Consequences
|
||||
|
||||
- The M7 RSS number now clears the DoD line: `heap-profile` reports 45.41 MiB for the
|
||||
20-task/8-segment/32-cap scenario (`core/docs/m7-baseline.md`), and
|
||||
`budget.active` never exceeds `budget.total` regardless of contention.
|
||||
- `docs/adr/0016`'s TSan-only load-test straggler (a *different* subsystem —
|
||||
`rate::RateLimiter`'s byte-pacing, not `SegmentBudget`'s segment-admission) is now
|
||||
suspected to have been this same root cause rather than the `CURLOPT_LOW_SPEED_TIME`
|
||||
guess offered there, since the symptom (a task that simply never resumes) matches
|
||||
exactly. Not reverified at the DoD's full shape under `--preset tsan` in this change —
|
||||
`tools/bench`'s sanitizer-preset `load` registration still runs at reduced concurrency.
|
||||
Worth a follow-up run before closing that ADR's postscript.
|
||||
- `docs/adr/0016`'s actual subject — `rate::RateLimiter::TokenBucket`'s own peek/commit
|
||||
race, unrelated to `SegmentBudget` — is untouched by this change and remains open.
|
||||
- `wake_one_waiter_locked()` wakes exactly one task per actual release, by priority order.
|
||||
Under sustained heavy oversubscription (this bench's 20 tasks × 8 segments = 160 wanted
|
||||
against 32 available) a low-priority task can still wait a long time for its fair share
|
||||
— that's the two-pass allocator's own fairness policy working as designed, not a
|
||||
liveness bug: it will get there, just not fast, and every fresh `reallocate_locked()`
|
||||
call (any task's `set_want`, registration, or departure) reconsiders everyone from
|
||||
scratch. No test in this change measures *how* long; if that ever needs a stronger
|
||||
guarantee (bounded wait time, not just eventual service), it's a fairness-policy change,
|
||||
not a wiring fix.
|
||||
|
||||
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
|
||||
@@ -0,0 +1,107 @@
|
||||
# ADR 0018 — Nullable optional fields: absent vs. explicit null
|
||||
|
||||
**Status:** accepted · **Date:** 2026-09-13 · **Lane:** PROTO
|
||||
**Prompted by:** a DAEMON report against `download.update`: the generated C++ parser gives
|
||||
`VeloxDispatcher` no way to tell "the caller left this field alone" from "the caller wants
|
||||
it cleared," so `download.update` and (the moment a nullable `SettingKey` exists)
|
||||
`settings.set` can set a nullable field but never clear it back to `null`.
|
||||
|
||||
## Context
|
||||
|
||||
`download.update`'s `patch` object documents the convention plainly: "Only the present
|
||||
fields change. An explicit null clears a nullable field." That is a deliberate, already-
|
||||
committed wire contract — not something up for redesign here. The gap is one layer down:
|
||||
`contracts/codegen/gen_cpp.py`'s `emit_field_parse` collapses "key absent" and "key present
|
||||
with value `null`" to the same `std::nullopt`, on purpose, and the comment says so:
|
||||
|
||||
> Absent and null mean the same thing: the field is not set. A client that omits a
|
||||
> nullable field and one that sends null are treated identically on purpose.
|
||||
|
||||
That collapse is *correct* for the common case — most nullable-optional fields are on
|
||||
create-style params (`DownloadSpec.saveDir`, `.categoryId`, …) where there is no existing
|
||||
value to distinguish "never set" from "explicitly cleared" in the first place; either way
|
||||
the daemon just uses a default. It is wrong specifically for **patch-style** params, where
|
||||
a field can already hold a value and the caller needs to say which of two different things
|
||||
they mean: "leave it" or "clear it."
|
||||
|
||||
The schema IR (`schema_ir.py`) already tracks `required` and `nullable` as two independent
|
||||
booleans per `Field`, so the information needed to make this distinction exists all the way
|
||||
through parsing — `emit_field_parse` just doesn't act on it. Only `download.update`'s
|
||||
`patch` object is affected today (`filename`, `saveDir`, `categoryId`, `queueId`,
|
||||
`description`, `segments`, `bufferBytes`, `checksum` — all eight of its fields are
|
||||
nullable-and-optional with exactly this "leave vs. clear" meaning). No `SettingKey` is
|
||||
nullable yet, so `settings.set` has no live instance of the bug, but the same shape
|
||||
(`values` patches an existing bag) means the first nullable settings key will hit the exact
|
||||
same gap.
|
||||
|
||||
## Decision
|
||||
|
||||
**A JSON-null-aware optional, opt in per field via a new `x-clearable: true` annotation —
|
||||
not a blanket rule and not a companion "clear list" field.**
|
||||
|
||||
- New per-field schema annotation, `x-clearable: true`, valid only on a field whose type
|
||||
already includes `null` (schema error otherwise — clearable implies nullable). Marks
|
||||
"this field distinguishes absent from explicit null"; every other nullable-optional field
|
||||
keeps today's collapse.
|
||||
- The generated C++ type for a `x-clearable` field becomes `std::optional<std::optional<T>>`:
|
||||
outer `nullopt` = absent (leave unchanged), outer engaged with an inner `nullopt` =
|
||||
explicit `null` (clear it), outer engaged with an inner value = set it. One field, three
|
||||
states, no parallel bitset to keep in sync and no second field to forget to check.
|
||||
- `emit_field_parse` for such a field stops folding `is_null()` into "absent": absent skips
|
||||
the assignment (outer stays `nullopt`); present-and-null assigns an engaged-but-empty
|
||||
inner optional; present-and-valued parses normally into the inner optional. Every other
|
||||
field's codegen (the `required`/`nullable`-but-not-`clearable` majority) is unchanged.
|
||||
- TypeScript needs no generator change: `field?: T | null` already round-trips this exactly
|
||||
the way JSON does — an omitted key serializes as absent, `null` serializes as `null`, and
|
||||
`"field" in obj` / `obj.field === null` already distinguish the three states natively.
|
||||
This gap is a C++-generator-only problem.
|
||||
- Applies now to `download.update`'s eight `patch` fields. `Settings` gets no annotation
|
||||
today (nothing nullable to mark); the day a nullable `SettingKey` is added, it gets
|
||||
`x-clearable: true` in the same PR, not left to rediscover this ADR.
|
||||
|
||||
## Versioning
|
||||
|
||||
Per ADR 0015: this retypes a generated C++ field (`optional<T>` -> `optional<optional<T>>`)
|
||||
with the wire byte-for-byte unchanged — a client sending the same JSON parses correctly
|
||||
either way. **Minor bump, with a migration note** for anyone reading `patch.filename` et al.
|
||||
directly (unwrap twice: check the outer, then the inner). Not major; `session.hello`'s
|
||||
major-only check must not refuse a wire-compatible peer over a binding-only change.
|
||||
|
||||
## Consequences
|
||||
|
||||
- `on_download_update` (DAEMON, not this lane) can finally implement "explicit null
|
||||
clears": read the outer optional for presence, the inner for clear-vs-value, exactly the
|
||||
three states the schema already promised.
|
||||
- The collapse comment in `emit_field_parse` stays as the default behavior and gets a
|
||||
pointer to this ADR for the opt-in exception, instead of being read as an oversight.
|
||||
- Implementation (schema annotation support in `schema_ir.py`, the `gen_cpp.py` emission
|
||||
change above, regenerating `core/generated/`, the `x-clearable: true` annotations on
|
||||
`download.update`'s eight fields, the VERSION bump and migration note) is **not** done in
|
||||
this change — recorded here so DAEMON isn't blocked on relitigating the design, tracked as
|
||||
its own PROTO PR per the normal contracts process (schema + regenerated code + fixtures +
|
||||
VERSION bump together, CLAUDE.md §2).
|
||||
|
||||
## Alternatives rejected
|
||||
|
||||
**An explicit clear list** (e.g. `patch.clearFields: ["categoryId", …]`, plain non-nullable
|
||||
`optional<T>` fields otherwise). Rejected: the wire contract "an explicit null clears a
|
||||
nullable field" is already written into `download.update`'s schema description and is what
|
||||
DAEMON built against — this would be a real, disruptive wire redesign to route around a
|
||||
generator gap, not a fix for it. It also doesn't compose: every patch-shaped object gains a
|
||||
second array to keep in sync with the first, by hand, forever.
|
||||
|
||||
**A parallel "which fields were present" bitset** (struct of `optional<T>` fields plus a
|
||||
sibling presence-flags struct or bitset). Rejected: two things to check per field instead
|
||||
of one, and nothing stops a caller from reading the optional and forgetting the presence
|
||||
bit — exactly the class of bug this ADR exists to close.
|
||||
|
||||
**Apply the tri-state to every `nullable && !required` field automatically**, using the IR
|
||||
flags already present, no annotation needed. Rejected: `emit_field_parse` only backs
|
||||
`parse<T>()`, used for *params* types the daemon receives — but the conformance C++ runner
|
||||
also instantiates `parse<T>()` for **result** types (round-tripping golden fixtures), and
|
||||
plenty of those are nullable-optional with no patch semantics at all (`TaskSummary.effectiveUrl`,
|
||||
"null until the first probe succeeds" — a plain nullable value, not a leave-or-clear
|
||||
choice). Blanket application would retype those too, forcing every read site across the
|
||||
daemon that already does `if (summary.effectiveUrl)` into an unwanted double-unwrap for a
|
||||
distinction that field doesn't have. Opt-in keeps the blast radius at exactly the fields
|
||||
that need it.
|
||||
@@ -0,0 +1,45 @@
|
||||
# ADR 0019 — The live-`veloxd` conformance runner is a required check, as-is
|
||||
|
||||
**Status:** accepted · **Date:** 2026-09-12 · **Lane:** PKG/QA
|
||||
|
||||
## Context
|
||||
|
||||
`tests/conformance/run.sh` step 3b (ADR 0014's `conformance` ctest, already required
|
||||
per `.github/BRANCH_PROTECTION.md`) replays every fixture against a real, isolated
|
||||
`veloxd` it builds and starts — not just mockd, which only proves the TS client and the
|
||||
fixtures agree with each other. This is the runner that can catch `veloxd` disagreeing
|
||||
with its own contract, and it is unconditional in `run.sh` (`set -euo pipefail`, no
|
||||
skip flag): it already runs, and already blocks, inside the `conformance` job.
|
||||
|
||||
What was open was whether to treat that as a settled, defended gate or as something
|
||||
still provisional while `daemon/docs/deferrals.md`'s D1-D4b stub handlers were excused
|
||||
via `veloxd-xfail.json`. PROTO's update: 57/57 fixtures pass on `main`, and the xfail
|
||||
allowlist is down to 18 entries from 34 as DAEMON lands the deferred handlers behind
|
||||
them.
|
||||
|
||||
## Decision
|
||||
|
||||
The live-`veloxd` conformance runner stays required — no change to CI is needed, since
|
||||
it already runs inside the already-required `conformance` job (ADR 0014). What this ADR
|
||||
records is the standing: PKG/QA is not carving out an exception, a `continue-on-error`,
|
||||
or a separate advisory job for it while the xfail list shrinks. A regression here fails
|
||||
the same required check a schema mismatch would.
|
||||
|
||||
Verified, not assumed: `tests/conformance/veloxd-xfail.json` has 18 entries as of this
|
||||
ADR (`python3 -c "import json; print(len(json.load(open('tests/conformance/veloxd-xfail.json'))))"`).
|
||||
Each remaining entry excuses one still-stubbed handler on `deferrals.md`'s D-list, not a
|
||||
real disagreement between `veloxd` and its contract — `tests/conformance/README.md`
|
||||
already draws that line (an entry for anything else is a `run.sh`-detected "xfail entry
|
||||
unexpectedly passed" or a straight failure, not a quiet pass).
|
||||
|
||||
## Consequences
|
||||
|
||||
- No `ci.yml` or `BRANCH_PROTECTION.md` change: `conformance` was already listed
|
||||
required, and this runner was already inside it.
|
||||
- The xfail list is a visible, shrinking number, not a static allowance — as DAEMON
|
||||
clears more of `deferrals.md`'s D-list, entries come out of
|
||||
`tests/conformance/veloxd-xfail.json`, and `replay.ts` fails loudly (per
|
||||
`applyXfail`'s "unexpectedly passed" check) if one is left in after its handler ships.
|
||||
- Nothing here changes who owns what: `veloxd-xfail.json` and `run.sh` stay PROTO's;
|
||||
PKG/QA's role is the branch-protection policy this ADR confirms, not the runner
|
||||
itself.
|
||||
@@ -0,0 +1,49 @@
|
||||
# AMO permission justification
|
||||
|
||||
Submitted with every version bump in the AMO "Notes to Reviewer" field. Kept here so the
|
||||
justification is reviewed and versioned alongside the permission list itself
|
||||
(`manifest.json`), instead of living only in a web form. docs/05 §7 is the design-time
|
||||
version of this; this file is the submission-ready copy.
|
||||
|
||||
## What Velox is
|
||||
|
||||
A download manager. It intercepts a response Firefox is about to download, hands the
|
||||
URL, request headers, and cookies to a companion native application (`veloxd`), and lets
|
||||
that application fetch the file with resumable, multi-connection transfers. The
|
||||
extension itself never stores or transfers file bytes — see `CLAUDE.md` §3 and the
|
||||
`no-download-logic` ESLint gate (`eslint.config.mjs`) that fails CI if it ever does.
|
||||
|
||||
## Permissions requested
|
||||
|
||||
| Permission | Why | Narrowest alternative considered |
|
||||
|---|---|---|
|
||||
| `webRequest` + `webRequestBlocking` | The whole feature: inspect response headers on `onHeadersReceived` to decide whether to intercept a download, and `{cancel: true}` before Firefox starts its own download. This is the one thing MV3 Chrome removed and MV3 Firefox kept — it's why the extension can exist as designed (docs/05 §1). | None. Without blocking `webRequest` there is no way to stop Firefox's own download before it starts; polling `downloads.onCreated` alone (which we also use, see below) only catches what already started. |
|
||||
| `downloads` | Belt-and-braces safety net (`capture/downloads-api.ts`): some downloads (form POSTs, service-worker blobs) never reach `onHeadersReceived` in a way we can act on and only surface via `downloads.onCreated`. Also used to `cancel`/`erase` a download we're taking over so Firefox doesn't keep two copies. | Drop the safety net and accept that those cases silently bypass Velox. Rejected — docs/05 §2 calls this out explicitly as a known gap the safety net exists to close. |
|
||||
| `cookies` | A file behind a login (private CDN links, forum attachments) needs its session cookies handed to `veloxd`, or the daemon's fetch gets a 403 the browser's own request wouldn't have. Read via `cookies.getAll(url)` only for a URL we are about to offer to the daemon — never harvested in bulk or logged. | Skip cookies and only support anonymous URLs. Rejected — it's a top user-facing IDM-parity feature and the reason people leave Chrome download managers behind. |
|
||||
| `contextMenus` | "Download with Velox" on a link/image/video, and "Download all links…" (docs/05 §3). Table-stakes UI for a download manager extension. | None smaller — there's no partial grant for context menus. |
|
||||
| `storage` | `browser.storage.local` holds only extension-local state: the WebSocket pairing token, the manual transport override, the last-good WS port, and the user's default-category preference (`transport/storage.ts`, `options/prefs.ts`). No browsing data. | None — some persistence is required for pairing to survive a restart (M1 DoD), which is the point of the token existing at all. |
|
||||
| `notifications` | Tells the user when the daemon can't be reached for a download that fell back to Firefox, and (native-messaging path) surfaces pairing prompts if the GUI isn't running. | Silent failure. Rejected — capture fails open by design (CLAUDE.md §4) and a silent fallback with no notification would look like a bug. |
|
||||
| `nativeMessaging` | Opportunistic transport to `veloxd` over a Unix socket, for installs where it works (docs/adr/0003). Not the default path — WebSocket is — but shipped because it avoids the WebSocket port-scan on installs where the native host manifest is reachable. | Drop native messaging and use WebSocket exclusively. Considered and rejected in ADR 0003: keeping both means the extension keeps working across deb/snap/flatpak Firefox without per-flavour capture-logic forks. |
|
||||
| `<all_urls>` (host permission) | Downloads happen from every site on the web; `webRequest`'s header inspection and `cookies.getAll` both need to run against whatever site the user is on. This is the item AMO reviewers push back on hardest for extensions of this shape. | A fixed list of "known download sites" — unworkable for a general-purpose download manager, and defeats the point of an IDM-style interceptor. **Mitigation, not a narrower permission:** the exclusion list in Options is front-and-center (`options.html` → "Capture policy") so a user can scope capture down to nothing on sites they don't want Velox touching, and the bypass modifier (default Alt) lets a single click skip capture without changing settings. |
|
||||
|
||||
## What is explicitly *not* requested
|
||||
|
||||
- No `<all_urls>` XHR/fetch use — `webRequest`/`cookies` read metadata about a request
|
||||
Firefox is already making; the extension never issues its own network request for
|
||||
file bytes (enforced by the ESLint gate above).
|
||||
- No `identity`, `history`, `bookmarks`, `tabs` beyond what `contextMenus`/`commands`
|
||||
already imply, `management`, or any permission unrelated to capturing and handing off
|
||||
a download.
|
||||
- No remote code: the manifest ships no CDN scripts and no `eval`; `web-ext lint` fails
|
||||
the build otherwise (CI's `extension-lint` job).
|
||||
|
||||
## Data handling
|
||||
|
||||
- Cookies and headers are held in memory only long enough to answer one
|
||||
`capture.offer` call to the local daemon (`capture/headers.ts`'s ring buffer, 5-minute
|
||||
TTL) — never written to disk by the extension and never sent anywhere but
|
||||
`127.0.0.1`.
|
||||
- The daemon connection is local-only: `WebSocketTransport` connects to
|
||||
`ws://127.0.0.1:<port>`, never a remote host (docs/05 §4, conformance-tested).
|
||||
- Nothing is sent to Anthropic, Mozilla, or any third party beyond the user's own local
|
||||
`veloxd` process.
|
||||
@@ -0,0 +1,90 @@
|
||||
// ESLint config for the extension.
|
||||
//
|
||||
// The "no download logic in extension/" rule (CLAUDE.md §3) used to be prose only.
|
||||
// GUI turned its half into a ctest (gui/tests/no_download_logic.cmake); this is EXT's
|
||||
// equivalent — a build-failing gate instead of something a reviewer has to remember to
|
||||
// look for. See gui/docs/ext-requests-m1.md for the request this answers.
|
||||
//
|
||||
// The extension's whole job is: collect URL + headers + cookies, hand them to veloxd,
|
||||
// render what comes back. It must never fetch bytes, assemble a Range request, or read
|
||||
// a response body itself — that is download logic, and it belongs in core/daemon only.
|
||||
import js from '@eslint/js';
|
||||
import tseslint from 'typescript-eslint';
|
||||
|
||||
const noDownloadLogic = {
|
||||
name: 'velox/no-download-logic',
|
||||
files: ['src/**/*.ts'],
|
||||
rules: {
|
||||
'no-restricted-syntax': [
|
||||
'error',
|
||||
{
|
||||
selector: "NewExpression[callee.name='XMLHttpRequest']",
|
||||
message:
|
||||
'No XMLHttpRequest in extension/ — the extension hands URLs to veloxd, it never fetches bytes itself (CLAUDE.md §3).',
|
||||
},
|
||||
{
|
||||
selector: "NewExpression[callee.name='Request']",
|
||||
message:
|
||||
'No `new Request(...)` in extension/ — that is download-side plumbing. Hand the URL to veloxd instead (CLAUDE.md §3).',
|
||||
},
|
||||
{
|
||||
selector: "CallExpression[callee.name='fetch']",
|
||||
message:
|
||||
'No fetch() in extension/ — the extension never retrieves download bytes itself (CLAUDE.md §3). Talking to veloxd goes through transport/, not fetch.',
|
||||
},
|
||||
{
|
||||
selector: "MemberExpression[property.name='getReader']",
|
||||
message:
|
||||
'No ReadableStream reader in extension/ — reading a response body here is download logic (CLAUDE.md §3); the daemon owns transfer bytes.',
|
||||
},
|
||||
{
|
||||
selector:
|
||||
"Property[key.name='Range'], Property[key.name='range'], Property[key.name='Content-Range'], Property[key.name='content-range']",
|
||||
message:
|
||||
'No Range/Content-Range header construction in extension/ — resumption is the daemon\'s job (CLAUDE.md §3, docs/05).',
|
||||
},
|
||||
{
|
||||
selector: "NewExpression[callee.object.name='indexedDB'], CallExpression[callee.object.name='indexedDB']",
|
||||
message: 'No IndexedDB in extension/ for moving bytes — hand off to veloxd instead (CLAUDE.md §3).',
|
||||
},
|
||||
],
|
||||
'no-restricted-globals': [
|
||||
'error',
|
||||
{ name: 'fetch', message: 'No fetch() in extension/ — see CLAUDE.md §3.' },
|
||||
{ name: 'XMLHttpRequest', message: 'No XMLHttpRequest in extension/ — see CLAUDE.md §3.' },
|
||||
{ name: 'indexedDB', message: 'No IndexedDB in extension/ — see CLAUDE.md §3.' },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
export default tseslint.config(
|
||||
{
|
||||
// src/shared/protocol/** is generated (contracts/codegen/gen_ts.py) and must never
|
||||
// be hand-edited — linting it as if we could fix a finding would be a lie.
|
||||
ignores: ['dist/**', 'node_modules/**', 'scripts/**', 'src/shared/protocol/**'],
|
||||
},
|
||||
js.configs.recommended,
|
||||
...tseslint.configs.recommended,
|
||||
{
|
||||
files: ['src/**/*.ts', 'tests/**/*.ts'],
|
||||
languageOptions: {
|
||||
parserOptions: {
|
||||
project: false,
|
||||
},
|
||||
},
|
||||
rules: {
|
||||
'@typescript-eslint/no-explicit-any': 'error',
|
||||
'@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
|
||||
},
|
||||
},
|
||||
noDownloadLogic,
|
||||
{
|
||||
// Tests legitimately construct fake Requests/fetch mocks to exercise transport code
|
||||
// against a fake server; the rule protects src/, not the harness that pokes at it.
|
||||
files: ['tests/**/*.ts'],
|
||||
rules: {
|
||||
'no-restricted-syntax': 'off',
|
||||
'no-restricted-globals': 'off',
|
||||
},
|
||||
},
|
||||
);
|
||||
@@ -19,6 +19,25 @@
|
||||
"type": "module"
|
||||
},
|
||||
|
||||
"action": {
|
||||
"default_popup": "dist/popup.html",
|
||||
"default_title": "Velox"
|
||||
},
|
||||
|
||||
"options_ui": {
|
||||
"page": "dist/options.html",
|
||||
"open_in_tab": true
|
||||
},
|
||||
|
||||
"content_scripts": [
|
||||
{
|
||||
"matches": ["<all_urls>"],
|
||||
"js": ["dist/content.js"],
|
||||
"run_at": "document_idle",
|
||||
"all_frames": true
|
||||
}
|
||||
],
|
||||
|
||||
"permissions": [
|
||||
"webRequest",
|
||||
"webRequestBlocking",
|
||||
|
||||
Generated
+920
-160
File diff suppressed because it is too large
Load Diff
@@ -10,13 +10,18 @@
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"lint": "web-ext lint --source-dir ."
|
||||
"lint": "npm run lint:eslint && npm run lint:webext",
|
||||
"lint:eslint": "eslint .",
|
||||
"lint:webext": "web-ext lint --source-dir ."
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/firefox-webext-browser": "^120.0.4",
|
||||
"@types/ws": "^8.5.12",
|
||||
"esbuild": "^0.24.0",
|
||||
"eslint": "^9.39.5",
|
||||
"happy-dom": "^15.11.7",
|
||||
"typescript": "^5.6.0",
|
||||
"typescript-eslint": "^8.70.0",
|
||||
"vitest": "^2.1.0",
|
||||
"web-ext": "^8.3.0",
|
||||
"ws": "^8.18.0"
|
||||
|
||||
@@ -5,29 +5,46 @@
|
||||
// build step. Firefox-only: no polyfill, native ESM, `browser.*` is a global.
|
||||
|
||||
import { build } from 'esbuild';
|
||||
import { rm, mkdir } from 'node:fs/promises';
|
||||
import { rm, mkdir, copyFile } from 'node:fs/promises';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
|
||||
const root = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const outdir = resolve(root, 'dist');
|
||||
|
||||
// One entry per manifest surface. Add popup/options/content here as they land.
|
||||
const entryPoints = {
|
||||
// The background page and the popup/options documents load as native ESM (manifest.json
|
||||
// declares "type": "module" for the background script; the popup/options HTML load
|
||||
// their script with type="module"). Content scripts registered via manifest.json's
|
||||
// content_scripts have no such declaration and run as classic scripts, so content.js is
|
||||
// built as an IIFE instead — an `export` left in by the esm build would be a syntax
|
||||
// error there.
|
||||
const esmEntryPoints = {
|
||||
background: resolve(root, 'src/background/index.ts'),
|
||||
popup: resolve(root, 'src/popup/popup.ts'),
|
||||
options: resolve(root, 'src/options/options.ts'),
|
||||
};
|
||||
const iifeEntryPoints = {
|
||||
content: resolve(root, 'src/content/index.ts'),
|
||||
};
|
||||
|
||||
// Static HTML/CSS esbuild doesn't touch — copied straight to dist/ alongside their JS.
|
||||
const staticFiles = [
|
||||
['src/popup/popup.html', 'popup.html'],
|
||||
['src/popup/popup.css', 'popup.css'],
|
||||
['src/options/options.html', 'options.html'],
|
||||
['src/options/options.css', 'options.css'],
|
||||
];
|
||||
|
||||
await rm(outdir, { recursive: true, force: true });
|
||||
await mkdir(outdir, { recursive: true });
|
||||
await Promise.all(staticFiles.map(([src, dest]) => copyFile(resolve(root, src), resolve(outdir, dest))));
|
||||
|
||||
const watch = process.argv.includes('--watch');
|
||||
const dev = watch || process.argv.includes('--dev');
|
||||
|
||||
const options = {
|
||||
entryPoints,
|
||||
const shared = {
|
||||
outdir,
|
||||
bundle: true,
|
||||
format: 'esm',
|
||||
target: ['firefox128'],
|
||||
platform: 'browser',
|
||||
sourcemap: dev ? 'inline' : 'linked',
|
||||
@@ -37,10 +54,16 @@ const options = {
|
||||
external: [],
|
||||
};
|
||||
|
||||
const buildConfigs = [
|
||||
{ ...shared, entryPoints: esmEntryPoints, format: 'esm' },
|
||||
{ ...shared, entryPoints: iifeEntryPoints, format: 'iife' },
|
||||
];
|
||||
|
||||
if (watch) {
|
||||
const ctx = await (await import('esbuild')).context(options);
|
||||
await ctx.watch();
|
||||
const { context } = await import('esbuild');
|
||||
const contexts = await Promise.all(buildConfigs.map((cfg) => context(cfg)));
|
||||
await Promise.all(contexts.map((ctx) => ctx.watch()));
|
||||
console.log('esbuild: watching');
|
||||
} else {
|
||||
await build(options);
|
||||
await Promise.all(buildConfigs.map((cfg) => build(cfg)));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
// Relays the background page's one VeloxTransport to the popup and options documents,
|
||||
// which run as separate contexts and cannot import background/index.ts directly.
|
||||
//
|
||||
// Wire protocol over a `browser.runtime.connect` port (see docs/05 §3: "live progress
|
||||
// via event.task.progress relayed over the transport"):
|
||||
// client -> bg { type: 'call', id, method, params }
|
||||
// client -> bg { type: 'subscribe', events: string[] } (replaces prior selection)
|
||||
// client -> bg { type: 'getStatus' }
|
||||
// bg -> client { type: 'result', id, ok: true, result } | { type: 'result', id, ok: false, error }
|
||||
// bg -> client { type: 'event', event, payload }
|
||||
// bg -> client { type: 'status', status }
|
||||
//
|
||||
// This is plain message relaying, not download logic: no bytes, no URLs fetched here,
|
||||
// just RPC forwarding to the transport the background page already owns.
|
||||
|
||||
import type { EventName, MethodName } from '../shared/protocol/index.js';
|
||||
import type { TransportKind, TransportStatus, VeloxTransport } from './transport/index.js';
|
||||
|
||||
/** TransportStatus plus which implementation is live — Options needs `kind` to know
|
||||
* whether pairing controls and privileged settings.set are meaningful right now. */
|
||||
export type PanelStatus = TransportStatus & { kind: TransportKind | null };
|
||||
|
||||
export type PanelRequest =
|
||||
| { type: 'call'; id: number; method: MethodName; params: unknown }
|
||||
| { type: 'subscribe'; events: EventName[] }
|
||||
| { type: 'getStatus' }
|
||||
| { type: 'reconnect' }
|
||||
| { type: 'pair'; code: string }
|
||||
| { type: 'unpair' }
|
||||
| { type: 'setOverride'; override: 'auto' | 'ws' | 'uds' };
|
||||
|
||||
export type PanelResponse =
|
||||
| { type: 'result'; id: number; ok: true; result: unknown }
|
||||
| { type: 'result'; id: number; ok: false; error: { code?: number; message: string } }
|
||||
| { type: 'event'; event: string; payload: unknown }
|
||||
| { type: 'status'; status: PanelStatus }
|
||||
| { type: 'pairError'; error: { code?: number; message: string } };
|
||||
|
||||
export interface PortLike {
|
||||
name: string;
|
||||
postMessage(message: PanelResponse): void;
|
||||
onMessage: { addListener(cb: (msg: PanelRequest) => void): void };
|
||||
onDisconnect: { addListener(cb: () => void): void };
|
||||
}
|
||||
|
||||
export interface RuntimeOnConnectLike {
|
||||
addListener(cb: (port: PortLike) => void): void;
|
||||
}
|
||||
|
||||
const DISCONNECTED_STATUS: PanelStatus = {
|
||||
state: 'disconnected',
|
||||
needsPairing: false,
|
||||
fatal: null,
|
||||
retryAfterSec: null,
|
||||
sessionId: null,
|
||||
daemonVersion: null,
|
||||
capabilities: [],
|
||||
kind: null,
|
||||
};
|
||||
|
||||
function panelStatus(t: VeloxTransport | undefined): PanelStatus {
|
||||
return t ? { ...t.status, kind: t.kind } : DISCONNECTED_STATUS;
|
||||
}
|
||||
|
||||
function errorOf(e: unknown): { code?: number; message: string } {
|
||||
if (e && typeof e === 'object') {
|
||||
const code = 'code' in e && typeof (e as { code: unknown }).code === 'number' ? (e as { code: number }).code : undefined;
|
||||
const message = e instanceof Error ? e.message : String(e);
|
||||
return code === undefined ? { message } : { code, message };
|
||||
}
|
||||
return { message: String(e) };
|
||||
}
|
||||
|
||||
export interface PanelBridgeDeps {
|
||||
getTransport(): VeloxTransport | undefined;
|
||||
/** Rebuilds the transport for a new manual override ('auto' lets the runtime picker
|
||||
* decide again) and swaps it in. Needed because switching kind means constructing a
|
||||
* different Transport implementation, not a method on the existing one. */
|
||||
setOverride(override: 'auto' | 'ws' | 'uds'): Promise<void>;
|
||||
}
|
||||
|
||||
export class PanelBridge {
|
||||
constructor(private readonly deps: PanelBridgeDeps) {}
|
||||
|
||||
private getTransport(): VeloxTransport | undefined {
|
||||
return this.deps.getTransport();
|
||||
}
|
||||
|
||||
attach(onConnect: RuntimeOnConnectLike): void {
|
||||
onConnect.addListener((port) => this.handleConnect(port));
|
||||
}
|
||||
|
||||
private handleConnect(port: PortLike): void {
|
||||
const unsubscribe: Array<() => void> = [];
|
||||
let disposed = false;
|
||||
|
||||
const post = (msg: PanelResponse): void => {
|
||||
if (!disposed) port.postMessage(msg);
|
||||
};
|
||||
|
||||
// The transport may not exist yet (background just woke up). Retry briefly rather
|
||||
// than leaving the panel stuck on "connecting" forever.
|
||||
const attachStatus = (attemptsLeft: number): void => {
|
||||
const t = this.getTransport();
|
||||
if (t) {
|
||||
post({ type: 'status', status: panelStatus(t) });
|
||||
const off = t.onStateChange(() => post({ type: 'status', status: panelStatus(this.getTransport()) }));
|
||||
unsubscribe.push(off);
|
||||
return;
|
||||
}
|
||||
post({ type: 'status', status: DISCONNECTED_STATUS });
|
||||
if (attemptsLeft > 0 && !disposed) {
|
||||
const timer = setTimeout(() => attachStatus(attemptsLeft - 1), 300);
|
||||
unsubscribe.push(() => clearTimeout(timer));
|
||||
}
|
||||
};
|
||||
attachStatus(10);
|
||||
|
||||
port.onMessage.addListener((msg) => void this.handleMessage(msg, post, unsubscribe));
|
||||
port.onDisconnect.addListener(() => {
|
||||
disposed = true;
|
||||
for (const u of unsubscribe) u();
|
||||
unsubscribe.length = 0;
|
||||
});
|
||||
}
|
||||
|
||||
private async handleMessage(
|
||||
msg: PanelRequest,
|
||||
post: (m: PanelResponse) => void,
|
||||
unsubscribe: Array<() => void>,
|
||||
): Promise<void> {
|
||||
if (msg.type === 'getStatus') {
|
||||
post({ type: 'status', status: panelStatus(this.getTransport()) });
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.type === 'reconnect') {
|
||||
// "Start it" in the popup (docs/05 §5) — never a fresh call the panel builds
|
||||
// params for itself; it just asks the transport it already owns to try again.
|
||||
this.getTransport()
|
||||
?.connect()
|
||||
.catch(() => undefined);
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.type === 'pair') {
|
||||
const t = this.getTransport();
|
||||
try {
|
||||
if (!t?.pairWithCode) throw new Error('pairing by code is only available on the WebSocket transport');
|
||||
await t.pairWithCode(msg.code);
|
||||
post({ type: 'status', status: panelStatus(t) });
|
||||
} catch (e) {
|
||||
post({ type: 'status', status: panelStatus(t) });
|
||||
post({ type: 'pairError', error: errorOf(e) });
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.type === 'unpair') {
|
||||
const t = this.getTransport();
|
||||
await t?.unpair?.();
|
||||
post({ type: 'status', status: panelStatus(this.getTransport()) });
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.type === 'setOverride') {
|
||||
await this.deps.setOverride(msg.override);
|
||||
post({ type: 'status', status: panelStatus(this.getTransport()) });
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.type === 'subscribe') {
|
||||
const t = this.getTransport();
|
||||
if (!t) return;
|
||||
for (const event of msg.events) {
|
||||
const cb = (payload: unknown) => post({ type: 'event', event, payload });
|
||||
t.on(event, cb);
|
||||
unsubscribe.push(() => t.off(event, cb));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// msg.type === 'call'
|
||||
const t = this.getTransport();
|
||||
if (!t) {
|
||||
post({ type: 'result', id: msg.id, ok: false, error: { message: 'transport not ready' } });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const result = await t.call(msg.method, msg.params as never);
|
||||
post({ type: 'result', id: msg.id, ok: true, result });
|
||||
} catch (e) {
|
||||
post({ type: 'result', id: msg.id, ok: false, error: errorOf(e) });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
// Streaming-media detection, background half. docs/05 §3: sniff `.m3u8`/`.mpd` and the
|
||||
// HLS/DASH content types on the wire, then tell the tab's content script a manifest was
|
||||
// seen. The extension never parses the manifest itself — media.listVariants (daemon-side)
|
||||
// does that; this module only recognizes "this response is probably a manifest".
|
||||
|
||||
const MANIFEST_EXTENSIONS = ['.m3u8', '.mpd'];
|
||||
const MANIFEST_CONTENT_TYPES = [
|
||||
'application/vnd.apple.mpegurl',
|
||||
'application/x-mpegurl',
|
||||
'audio/mpegurl',
|
||||
'application/dash+xml',
|
||||
];
|
||||
|
||||
/** Extension check ignores the query string — `manifest.m3u8?token=…` still counts. */
|
||||
export function isManifestUrl(url: string): boolean {
|
||||
let pathname: string;
|
||||
try {
|
||||
pathname = new URL(url).pathname.toLowerCase();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
return MANIFEST_EXTENSIONS.some((ext) => pathname.endsWith(ext));
|
||||
}
|
||||
|
||||
export function isManifestContentType(contentType: string | null | undefined): boolean {
|
||||
if (!contentType) return false;
|
||||
const base = contentType.split(';')[0]!.trim().toLowerCase();
|
||||
return MANIFEST_CONTENT_TYPES.includes(base);
|
||||
}
|
||||
|
||||
export function looksLikeManifest(url: string, contentType: string | null | undefined): boolean {
|
||||
return isManifestUrl(url) || isManifestContentType(contentType);
|
||||
}
|
||||
|
||||
export interface DetectedManifest {
|
||||
tabId: number;
|
||||
url: string;
|
||||
headers: Array<{ name: string; value: string }>;
|
||||
}
|
||||
|
||||
interface HeadersReceivedDetails {
|
||||
tabId: number;
|
||||
url: string;
|
||||
responseHeaders?: Array<{ name: string; value: string }>;
|
||||
}
|
||||
|
||||
export interface MediaWebRequest {
|
||||
onHeadersReceived: {
|
||||
addListener(
|
||||
cb: (details: HeadersReceivedDetails) => void,
|
||||
filter: { urls: string[]; types?: string[] },
|
||||
extraInfoSpec?: string[],
|
||||
): void;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Watches responses for HLS/DASH manifests and reports each hit once per (tab, url) —
|
||||
* a page can request the same manifest repeatedly (HLS live-refresh) and the content
|
||||
* script only needs to hear about it once to show the panel.
|
||||
*/
|
||||
export class MediaWatcher {
|
||||
private readonly seen = new Set<string>();
|
||||
|
||||
constructor(private readonly onDetected: (m: DetectedManifest) => void) {}
|
||||
|
||||
attach(webRequest: MediaWebRequest): void {
|
||||
webRequest.onHeadersReceived.addListener(
|
||||
(details) => this.handle(details),
|
||||
{ urls: ['<all_urls>'] },
|
||||
['responseHeaders'],
|
||||
);
|
||||
}
|
||||
|
||||
private handle(details: HeadersReceivedDetails): void {
|
||||
if (details.tabId < 0) return; // not a request associated with any tab
|
||||
const headers = details.responseHeaders ?? [];
|
||||
const contentType = headers.find((h) => h.name.toLowerCase() === 'content-type')?.value;
|
||||
if (!looksLikeManifest(details.url, contentType)) return;
|
||||
|
||||
const key = `${details.tabId}:${details.url}`;
|
||||
if (this.seen.has(key)) return;
|
||||
this.seen.add(key);
|
||||
|
||||
this.onDetected({ tabId: details.tabId, url: details.url, headers });
|
||||
}
|
||||
}
|
||||
@@ -4,9 +4,11 @@
|
||||
// paths: the blocking onHeadersReceived hook and the downloads.onCreated safety net. The
|
||||
// popup relay and context menus attach here in later steps of the build order.
|
||||
|
||||
import { PanelBridge } from './bridge.js';
|
||||
import { DownloadsSafetyNet, type DownloadsApiLike } from './capture/downloads-api.js';
|
||||
import { HeaderStash, type WebRequestLike } from './capture/headers.js';
|
||||
import { CaptureHook, type HeadersReceivedWebRequest } from './capture/index.js';
|
||||
import { MediaWatcher, type MediaWebRequest } from './capture/media.js';
|
||||
import { OfferedUrls } from './capture/offered-urls.js';
|
||||
import { DEFAULT_CAPTURE_RULES } from './capture/rules.js';
|
||||
import {
|
||||
@@ -15,8 +17,14 @@ import {
|
||||
type MenusLike,
|
||||
type TabsLike,
|
||||
} from './context-menus.js';
|
||||
import { createTransport, type TransportStatus, type VeloxTransport } from './transport/index.js';
|
||||
import type { CaptureOfferParams, CaptureRules, DownloadSpec } from '../shared/protocol/index.js';
|
||||
import { MediaBridge, notifyTab } from './media-bridge.js';
|
||||
import { createTransport, transportStorage, type TransportStatus, type VeloxTransport } from './transport/index.js';
|
||||
import {
|
||||
SESSION_SUBSCRIBE_PARAMS_EVENTS_ITEM_VALUES,
|
||||
type CaptureOfferParams,
|
||||
type CaptureRules,
|
||||
type DownloadSpec,
|
||||
} from '../shared/protocol/index.js';
|
||||
|
||||
let transport: VeloxTransport | undefined;
|
||||
let rules: CaptureRules = DEFAULT_CAPTURE_RULES;
|
||||
@@ -72,10 +80,49 @@ async function refreshRules(): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* "Nothing is delivered until this is called" (session.subscribe's own description) —
|
||||
* without it, event.task.progress et al. never reach this connection at all, no matter
|
||||
* how many listeners bridge.ts registers locally. Requests the whole set every time
|
||||
* because any popup/options document could open at any moment and none of them narrow
|
||||
* per-tab; a fresh connection (first connect, or after a drop) starts with nothing
|
||||
* subscribed until this runs again.
|
||||
*/
|
||||
async function subscribeToEvents(): Promise<void> {
|
||||
try {
|
||||
await mustTransport().call('session.subscribe', {
|
||||
events: [...SESSION_SUBSCRIBE_PARAMS_EVENTS_ITEM_VALUES],
|
||||
});
|
||||
} catch {
|
||||
// Best-effort; a reconnect (or the next event.settings.changed-driven refresh) retries.
|
||||
}
|
||||
}
|
||||
|
||||
function onTransportState(status: TransportStatus): void {
|
||||
const detail = status.fatal ?? (status.needsPairing ? 'needs pairing' : '');
|
||||
console.debug(`[velox] transport ${status.state}${detail ? ` — ${detail}` : ''}`);
|
||||
if (status.state === 'connected') void refreshRules();
|
||||
if (status.state === 'connected') {
|
||||
void refreshRules();
|
||||
void subscribeToEvents();
|
||||
}
|
||||
}
|
||||
|
||||
async function setOverride(override: 'auto' | 'ws' | 'uds'): Promise<void> {
|
||||
await transportStorage.setOverride(override);
|
||||
transport?.disconnect();
|
||||
transport = await createTransport({ override });
|
||||
transport.onStateChange(onTransportState);
|
||||
transport.on('event.settings.changed', onSettingsChanged);
|
||||
onTransportState(transport.status);
|
||||
}
|
||||
|
||||
const bridge = new PanelBridge({ getTransport: () => transport, setOverride });
|
||||
const mediaBridge = new MediaBridge(() => transport);
|
||||
const mediaWatcher = new MediaWatcher((detected) => notifyTab(browser.tabs, detected));
|
||||
|
||||
function onSettingsChanged(payload: unknown): void {
|
||||
const keys = (payload as { keys?: string[] }).keys ?? [];
|
||||
if (keys.some((k) => k.startsWith('capture.'))) void refreshRules();
|
||||
}
|
||||
|
||||
async function start(): Promise<void> {
|
||||
@@ -83,13 +130,13 @@ async function start(): Promise<void> {
|
||||
hook.attach(browser.webRequest as unknown as HeadersReceivedWebRequest);
|
||||
safetyNet.attach(browser.downloads as unknown as DownloadsApiLike);
|
||||
void contextMenus.register();
|
||||
bridge.attach(browser.runtime.onConnect);
|
||||
mediaBridge.attach(browser.runtime.onMessage);
|
||||
mediaWatcher.attach(browser.webRequest as unknown as MediaWebRequest);
|
||||
|
||||
transport = await createTransport();
|
||||
transport.onStateChange(onTransportState);
|
||||
transport.on('event.settings.changed', (payload) => {
|
||||
const keys = (payload as { keys?: string[] }).keys ?? [];
|
||||
if (keys.some((k) => k.startsWith('capture.'))) void refreshRules();
|
||||
});
|
||||
transport.on('event.settings.changed', onSettingsChanged);
|
||||
onTransportState(transport.status);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
// Wires capture/media.ts's manifest detection to the content scripts, and answers the
|
||||
// two calls the in-page video panel needs (media.listVariants, media.addVariant). A
|
||||
// separate, simpler channel from bridge.ts's port because content scripts are per-tab
|
||||
// and use one-shot runtime.sendMessage rather than a long-lived port.
|
||||
|
||||
import type { MediaAddVariantParams, MediaListVariantsParams } from '../shared/protocol/index.js';
|
||||
import type { VeloxTransport } from './transport/index.js';
|
||||
import type { DetectedManifest } from './capture/media.js';
|
||||
|
||||
export type ContentMessage =
|
||||
| { type: 'velox-list-variants'; params: MediaListVariantsParams }
|
||||
| { type: 'velox-add-variant'; params: MediaAddVariantParams };
|
||||
|
||||
export type ManifestDetectedMessage = { type: 'velox-manifest-detected'; url: string; headers: DetectedManifest['headers'] };
|
||||
|
||||
export interface TabsLike {
|
||||
sendMessage(tabId: number, message: unknown): Promise<unknown>;
|
||||
}
|
||||
|
||||
export interface RuntimeOnMessageLike {
|
||||
addListener(cb: (msg: ContentMessage, sender: unknown, sendResponse: (r: unknown) => void) => boolean | void): void;
|
||||
}
|
||||
|
||||
/** Tells the tab's content script a manifest was seen, swallowing the "no content
|
||||
* script listening" error a tab without one (or the daemon's own tab) throws. */
|
||||
export function notifyTab(tabs: TabsLike, manifest: DetectedManifest): void {
|
||||
const message: ManifestDetectedMessage = { type: 'velox-manifest-detected', url: manifest.url, headers: manifest.headers };
|
||||
tabs.sendMessage(manifest.tabId, message).catch(() => undefined);
|
||||
}
|
||||
|
||||
export class MediaBridge {
|
||||
constructor(private readonly getTransport: () => VeloxTransport | undefined) {}
|
||||
|
||||
attach(onMessage: RuntimeOnMessageLike): void {
|
||||
onMessage.addListener((msg, _sender, sendResponse) => {
|
||||
if (msg.type !== 'velox-list-variants' && msg.type !== 'velox-add-variant') return undefined;
|
||||
void this.handle(msg).then(sendResponse);
|
||||
return true; // sendResponse is called asynchronously
|
||||
});
|
||||
}
|
||||
|
||||
private async handle(msg: ContentMessage): Promise<{ ok: true; result: unknown } | { ok: false; error: string }> {
|
||||
const t = this.getTransport();
|
||||
if (!t) return { ok: false, error: 'transport not ready' };
|
||||
try {
|
||||
const result =
|
||||
msg.type === 'velox-list-variants'
|
||||
? await t.call('media.listVariants', msg.params)
|
||||
: await t.call('media.addVariant', msg.params);
|
||||
return { ok: true, result };
|
||||
} catch (e) {
|
||||
return { ok: false, error: e instanceof Error ? e.message : String(e) };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -62,6 +62,11 @@ export interface VeloxTransport {
|
||||
|
||||
/** Fires on every state change. Returns an unsubscribe. */
|
||||
onStateChange(cb: (status: TransportStatus) => void): () => void;
|
||||
|
||||
/** WebSocket transport only (pairing has no meaning over native messaging's uds
|
||||
* socket, which has no token). Options renders these controls only when present. */
|
||||
pairWithCode?(code: string): Promise<void>;
|
||||
unpair?(): Promise<void>;
|
||||
}
|
||||
|
||||
// --- errors ---------------------------------------------------------------------------
|
||||
|
||||
@@ -97,6 +97,9 @@ export class WebSocketTransport implements VeloxTransport {
|
||||
private stopped = false;
|
||||
private connectPromise: Promise<void> | null = null;
|
||||
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
/** Set only for the duration of pairWithCode(); consumed by pair(). docs/05 §4: "the
|
||||
* user clicks Allow (or types the code in the extension options)." */
|
||||
private pendingPairCode: string | null = null;
|
||||
|
||||
private readonly listeners = new Map<string, Set<EventListener>>();
|
||||
private readonly stateListeners = new Set<(status: TransportStatus) => void>();
|
||||
@@ -139,6 +142,29 @@ export class WebSocketTransport implements VeloxTransport {
|
||||
return this.connectPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options → "Pair" with a code typed from the daemon's dialog, for when the GUI isn't
|
||||
* running to click Allow (docs/05 §4). Drops any stored token first so the handshake
|
||||
* takes the pairing branch, then reconnects with the code attached.
|
||||
*/
|
||||
async pairWithCode(code: string): Promise<void> {
|
||||
this.disconnect();
|
||||
await this.deps.setToken(null); // drop any stale token so the handshake takes the pairing branch
|
||||
this.pendingPairCode = code;
|
||||
try {
|
||||
await this.connect();
|
||||
} finally {
|
||||
this.pendingPairCode = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Options → "Unpair": revoke the local token. The daemon's own record of it is
|
||||
* cleaned up on its side; this only ever forgets our copy. */
|
||||
async unpair(): Promise<void> {
|
||||
await this.deps.setToken(null);
|
||||
this.disconnect();
|
||||
}
|
||||
|
||||
disconnect(): void {
|
||||
this.stopped = true;
|
||||
if (this.reconnectTimer) {
|
||||
@@ -322,6 +348,7 @@ export class WebSocketTransport implements VeloxTransport {
|
||||
const params: SessionPairParams = {
|
||||
clientName: this.clientName,
|
||||
extensionId: this.deps.extensionId,
|
||||
code: this.pendingPairCode,
|
||||
};
|
||||
try {
|
||||
const res = (await rpc.request(
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
// Content script entry: registered for <all_urls> in manifest.json. Watches for
|
||||
// streaming media (docs/05 §3) and shows the "Download this video ▾" panel when either
|
||||
// half of detection fires — the background webRequest sniffer (capture/media.ts, relayed
|
||||
// via media-bridge.ts) or this page's own <video> elements (media-observer.ts).
|
||||
//
|
||||
// No download logic here either: this only asks the background page to call
|
||||
// media.listVariants / media.addVariant and renders what comes back.
|
||||
|
||||
import type { MediaAddVariantResult, MediaListVariantsResult } from '../shared/protocol/index.js';
|
||||
import { VideoSrcObserver } from './media-observer.js';
|
||||
import { VideoPanel, type VariantsPort } from './video-panel.js';
|
||||
|
||||
interface ManifestDetectedMessage {
|
||||
type: 'velox-manifest-detected';
|
||||
url: string;
|
||||
headers: Array<{ name: string; value: string }>;
|
||||
}
|
||||
|
||||
const headersByManifest = new Map<string, Array<{ name: string; value: string }>>();
|
||||
|
||||
const port: VariantsPort = {
|
||||
async listVariants(manifestUrl) {
|
||||
const headers = headersByManifest.get(manifestUrl);
|
||||
const response = (await browser.runtime.sendMessage({
|
||||
type: 'velox-list-variants',
|
||||
params: { manifestUrl, headers: headers?.length ? headers : null },
|
||||
})) as { ok: true; result: MediaListVariantsResult } | { ok: false; error: string };
|
||||
return response.ok ? response.result : { error: response.error };
|
||||
},
|
||||
async addVariant(manifestUrl, variantId) {
|
||||
const response = (await browser.runtime.sendMessage({
|
||||
type: 'velox-add-variant',
|
||||
params: { manifestUrl, variantId },
|
||||
})) as { ok: true; result: MediaAddVariantResult } | { ok: false; error: string };
|
||||
return response.ok ? { taskId: response.result.taskId } : { error: response.error };
|
||||
},
|
||||
};
|
||||
|
||||
const panel = new VideoPanel(port);
|
||||
|
||||
function onManifestSeen(url: string): void {
|
||||
panel.show(url);
|
||||
}
|
||||
|
||||
browser.runtime.onMessage.addListener((msg: unknown) => {
|
||||
const m = msg as Partial<ManifestDetectedMessage>;
|
||||
if (m.type !== 'velox-manifest-detected' || typeof m.url !== 'string') return undefined;
|
||||
headersByManifest.set(m.url, m.headers ?? []);
|
||||
onManifestSeen(m.url);
|
||||
return undefined;
|
||||
});
|
||||
|
||||
const observer = new VideoSrcObserver((url) => onManifestSeen(url));
|
||||
observer.start();
|
||||
@@ -0,0 +1,65 @@
|
||||
// DOM half of streaming-media detection (docs/05 §3: "a content script observing
|
||||
// MediaSource.addSourceBuffer and <video> src changes"). Pure DOM watching, no network:
|
||||
// the background half (capture/media.ts) is what actually inspects responses. This
|
||||
// module only recognizes "a <video> on this page points at something that smells like
|
||||
// an HLS/DASH manifest" and reports the URL up to whoever is watching (content/index.ts).
|
||||
|
||||
const MANIFEST_EXTENSIONS = ['.m3u8', '.mpd'];
|
||||
|
||||
function looksLikeManifestUrl(url: string): boolean {
|
||||
try {
|
||||
const pathname = new URL(url, document.baseURI).pathname.toLowerCase();
|
||||
return MANIFEST_EXTENSIONS.some((ext) => pathname.endsWith(ext));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Watches every <video> on the page (present now or added/changed later) for a src
|
||||
* that looks like a manifest URL, and reports each distinct URL once. */
|
||||
export class VideoSrcObserver {
|
||||
private readonly seen = new Set<string>();
|
||||
private mutationObserver: MutationObserver | null = null;
|
||||
|
||||
constructor(private readonly onCandidate: (url: string) => void) {}
|
||||
|
||||
start(root: ParentNode = document): void {
|
||||
this.scan(root);
|
||||
this.mutationObserver = new MutationObserver((mutations) => {
|
||||
for (const m of mutations) {
|
||||
if (m.type === 'attributes' && m.target instanceof HTMLVideoElement) {
|
||||
this.check(m.target);
|
||||
}
|
||||
for (const node of m.addedNodes) {
|
||||
if (node instanceof HTMLVideoElement) this.check(node);
|
||||
else if (node instanceof Element) this.scan(node);
|
||||
}
|
||||
}
|
||||
});
|
||||
this.mutationObserver.observe(document.documentElement ?? document, {
|
||||
subtree: true,
|
||||
childList: true,
|
||||
attributes: true,
|
||||
attributeFilter: ['src'],
|
||||
});
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.mutationObserver?.disconnect();
|
||||
this.mutationObserver = null;
|
||||
}
|
||||
|
||||
private scan(root: ParentNode): void {
|
||||
for (const video of root.querySelectorAll('video')) this.check(video);
|
||||
if (root instanceof HTMLVideoElement) this.check(root);
|
||||
}
|
||||
|
||||
private check(video: HTMLVideoElement): void {
|
||||
const candidates = [video.currentSrc, video.src].filter(Boolean);
|
||||
for (const url of candidates) {
|
||||
if (this.seen.has(url) || !looksLikeManifestUrl(url)) continue;
|
||||
this.seen.add(url);
|
||||
this.onCandidate(url);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
// In-page "Download this video ▾" panel (docs/05 §3). Lists variants the daemon
|
||||
// enumerated via media.listVariants — the extension never parses the manifest itself.
|
||||
// DRM-protected variants (and a wholly DRM-protected manifest) are shown greyed out
|
||||
// with "Protected content" rather than attempted.
|
||||
|
||||
import type { MediaVariant } from '../shared/protocol/index.js';
|
||||
|
||||
export interface VariantsPort {
|
||||
listVariants(manifestUrl: string): Promise<{ variants: MediaVariant[]; drmProtected: boolean } | { error: string }>;
|
||||
addVariant(manifestUrl: string, variantId: string): Promise<{ taskId: string } | { error: string }>;
|
||||
}
|
||||
|
||||
function variantLabel(v: MediaVariant): string {
|
||||
const bits = [v.resolution, v.codec, v.bitrateBps ? `${Math.round(v.bitrateBps / 1000)} kbps` : null].filter(Boolean);
|
||||
return bits.length > 0 ? bits.join(' · ') : v.variantId;
|
||||
}
|
||||
|
||||
const PANEL_ID = 'velox-video-panel';
|
||||
|
||||
export class VideoPanel {
|
||||
private root: HTMLElement | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly port: VariantsPort,
|
||||
private readonly doc: Document = document,
|
||||
) {}
|
||||
|
||||
/** Shows (or replaces) the floating panel for one detected manifest URL. */
|
||||
show(manifestUrl: string): void {
|
||||
this.remove();
|
||||
|
||||
const root = this.doc.createElement('div');
|
||||
root.id = PANEL_ID;
|
||||
Object.assign(root.style, {
|
||||
position: 'fixed',
|
||||
bottom: '16px',
|
||||
right: '16px',
|
||||
zIndex: '2147483647',
|
||||
font: '13px sans-serif',
|
||||
} satisfies Partial<CSSStyleDeclaration>);
|
||||
|
||||
const button = this.doc.createElement('button');
|
||||
button.textContent = 'Download this video ▾';
|
||||
root.appendChild(button);
|
||||
|
||||
const menu = this.doc.createElement('ul');
|
||||
menu.hidden = true;
|
||||
Object.assign(menu.style, { listStyle: 'none', margin: '4px 0 0', padding: '4px' } satisfies Partial<CSSStyleDeclaration>);
|
||||
root.appendChild(menu);
|
||||
|
||||
button.addEventListener('click', () => {
|
||||
menu.hidden = !menu.hidden;
|
||||
if (!menu.hidden) void this.populate(menu, manifestUrl);
|
||||
});
|
||||
|
||||
this.doc.body.appendChild(root);
|
||||
this.root = root;
|
||||
}
|
||||
|
||||
remove(): void {
|
||||
this.root?.remove();
|
||||
this.root = null;
|
||||
}
|
||||
|
||||
private async populate(menu: HTMLUListElement, manifestUrl: string): Promise<void> {
|
||||
menu.replaceChildren(this.loadingItem());
|
||||
const result = await this.port.listVariants(manifestUrl);
|
||||
|
||||
if ('error' in result) {
|
||||
menu.replaceChildren(this.messageItem(`Could not read variants: ${result.error}`));
|
||||
return;
|
||||
}
|
||||
if (result.drmProtected || result.variants.length === 0) {
|
||||
menu.replaceChildren(this.messageItem(result.drmProtected ? 'Protected content' : 'No downloadable variants found'));
|
||||
return;
|
||||
}
|
||||
|
||||
menu.replaceChildren(
|
||||
...result.variants.map((v) => {
|
||||
const li = this.doc.createElement('li');
|
||||
const item = this.doc.createElement('button');
|
||||
item.textContent = v.drm ? `${variantLabel(v)} — Protected content` : variantLabel(v);
|
||||
item.disabled = v.drm;
|
||||
if (!v.drm) {
|
||||
item.addEventListener('click', () => void this.port.addVariant(manifestUrl, v.variantId));
|
||||
}
|
||||
li.appendChild(item);
|
||||
return li;
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
private loadingItem(): HTMLLIElement {
|
||||
return this.messageItem('Loading…');
|
||||
}
|
||||
|
||||
private messageItem(text: string): HTMLLIElement {
|
||||
const li = this.doc.createElement('li');
|
||||
li.textContent = text;
|
||||
return li;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
body {
|
||||
max-width: 560px;
|
||||
margin: 0 auto;
|
||||
padding: 24px 16px;
|
||||
font: 14px -apple-system, system-ui, sans-serif;
|
||||
color: #1a1a1a;
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
h1 { margin: 0 0 16px; }
|
||||
|
||||
section {
|
||||
margin-bottom: 28px;
|
||||
padding-bottom: 20px;
|
||||
border-bottom: 1px solid #eee;
|
||||
}
|
||||
section:last-of-type { border-bottom: none; }
|
||||
|
||||
h2 {
|
||||
font-size: 15px;
|
||||
margin: 0 0 10px;
|
||||
}
|
||||
|
||||
label {
|
||||
display: block;
|
||||
margin: 8px 0;
|
||||
font-weight: 600;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
select, input, textarea {
|
||||
display: block;
|
||||
margin-top: 4px;
|
||||
font: inherit;
|
||||
padding: 5px 7px;
|
||||
width: 100%;
|
||||
max-width: 320px;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
button {
|
||||
font: inherit;
|
||||
padding: 6px 12px;
|
||||
margin-top: 8px;
|
||||
margin-right: 8px;
|
||||
}
|
||||
|
||||
#rules-locked {
|
||||
color: #888;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
#pair-message {
|
||||
min-height: 1.2em;
|
||||
color: #555;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user