Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e0f22f808f | ||
|
|
111229686c | ||
|
|
a5551db1c3 | ||
|
|
83b92d118b | ||
|
|
78f2b22c8c | ||
|
|
77bdb0c427 | ||
|
|
d60cf94905 | ||
|
|
0cd5bf6882 | ||
|
|
31eadffce5 | ||
|
|
5b42e1f0e5 | ||
|
|
e8af9b46b1 | ||
|
|
8abfc6a428 | ||
|
|
936136db06 | ||
|
|
6d7c4f7bc4 | ||
|
|
9487202ea8 | ||
|
|
9624e685af | ||
|
|
4e87730ce9 | ||
|
|
facd851824 | ||
|
|
eb72aa522c | ||
|
|
3e553f08c0 | ||
|
|
1d359af5a3 | ||
|
|
75536304bb | ||
|
|
27865888be | ||
|
|
939a19b7ea | ||
|
|
79c29b47e8 | ||
|
|
8e7d14ba7e | ||
|
|
c2eef96175 | ||
|
|
1fd2e0a0db | ||
|
|
755d85964e | ||
|
|
c6f864ea30 | ||
|
|
c1c5c82f8b | ||
|
|
4e177ec809 | ||
|
|
b5c6e1f47d | ||
|
|
0468b0176a | ||
|
|
d8b7c128be | ||
|
|
4f6c0cc9d2 | ||
|
|
e30d994d74 | ||
|
|
6632b75099 | ||
|
|
e0edf7a084 | ||
|
|
f4a6cebb3e | ||
|
|
963a76b6be | ||
|
|
765c1701d4 | ||
|
|
b578e6de1a | ||
|
|
ff721f4065 | ||
|
|
f2ee45f818 | ||
|
|
322a20efa5 | ||
|
|
7ea04fa79c | ||
|
|
ef58796d22 | ||
|
|
cf9e226e61 | ||
|
|
c89158ea09 | ||
|
|
4f6fb1029d | ||
|
|
55f0c6099d | ||
|
|
de83ee3cce | ||
|
|
39c69f3871 | ||
|
|
efe76c319d | ||
|
|
de748cc2fc | ||
|
|
67aefbce4b | ||
|
|
4df4d7ad08 | ||
|
|
f73a57a9ad | ||
|
|
c710481018 | ||
|
|
d4ad48d494 | ||
|
|
b60d4e6f5b | ||
|
|
6163898c14 | ||
|
|
ef816c21fb | ||
|
|
c99d1d9701 | ||
|
|
5b03eff926 | ||
|
|
03b6253b5a | ||
|
|
55932a2e11 | ||
|
|
25c171f742 | ||
|
|
2cb1959bff | ||
|
|
a967eca669 | ||
|
|
a71d904a1f | ||
|
|
41bce91770 | ||
|
|
0a38867579 | ||
|
|
0af4a5c4fc | ||
|
|
ba29fcb5bd | ||
|
|
824fa481bb |
@@ -18,8 +18,9 @@ policy so it can be re-applied or audited.
|
||||
| `bootstrap-script-2604` | now — real `--with-clang` install in a 26.04 container; the release the project ships on |
|
||||
| `build (gcc)` / `build (clang)` | now — core, daemon and gui have merged |
|
||||
| `sanitizers (dev)` / `sanitizers (tsan)` | now — core, daemon and gui have merged |
|
||||
| `conformance` | **now — `tests/conformance/` has landed; this is the M0 exit gate** |
|
||||
| `conformance` | **now — `tests/conformance/` has landed; this is the M0 exit gate.** Includes the live-`veloxd` runner (step 3b of `run.sh`), unconditional in the script — see `docs/adr/0019-live-veloxd-conformance-is-required.md`. |
|
||||
| `extension-lint` | now — `extension/` has merged (MV3 manifest + esbuild build) |
|
||||
| `gui-dod` | now — `gui/tests/dod/` has landed (GUI M1 DoD gates R3: `scroll-60fps`, `unhappy-path`); see `tests/integration/README.md#gui-m1-definition-of-done-gates-r3`. `gui-dod-nightly` (`rss-flat`) is schedule-only and cannot be a required PR check. |
|
||||
|
||||
`clang-tidy` is intentionally **not** required through M1 (`continue-on-error: true`,
|
||||
`.clang-tidy` has `WarningsAsErrors: ''`). Make it required at M2.
|
||||
|
||||
@@ -4,6 +4,9 @@ on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
schedule:
|
||||
- cron: '17 3 * * *' # nightly-integration only; every other job stays PR/push-triggered
|
||||
workflow_dispatch: # lets a human fire nightly-integration on demand
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
@@ -105,11 +108,16 @@ jobs:
|
||||
if: steps.check.outputs.present == 'true'
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: web-ext lint
|
||||
- name: eslint (no-download-logic gate + general rules)
|
||||
if: steps.check.outputs.present == 'true'
|
||||
working-directory: extension
|
||||
run: |
|
||||
npm ci
|
||||
npx eslint .
|
||||
- name: web-ext lint
|
||||
if: steps.check.outputs.present == 'true'
|
||||
working-directory: extension
|
||||
run: |
|
||||
npx web-ext lint --source-dir .
|
||||
|
||||
# --- build + test matrix ----------------------------------------------------------
|
||||
@@ -220,3 +228,76 @@ jobs:
|
||||
run: cmake --build --preset dev --target velox_conformance_cpp
|
||||
- name: Run conformance (ctest -L conformance)
|
||||
run: ctest --preset dev -L conformance --output-on-failure
|
||||
|
||||
nightly-integration:
|
||||
# Real veloxd + tools/testserver, 50 concurrent downloads mixing hostile modes,
|
||||
# every completed file's SHA-256 checked against testserver's own /sha256/ route,
|
||||
# veloxd's open-FD count checked flat across the run. Nightly, not per-PR: it's
|
||||
# ~2 minutes of real network I/O against a local server, not a schema check.
|
||||
# See tests/integration/README.md#nightly-integration-run for what each assertion
|
||||
# catches and the forced-failure transcript proving it isn't vacuous.
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Bootstrap toolchain
|
||||
run: sudo ./tools/bootstrap.sh
|
||||
- name: Configure
|
||||
run: cmake --preset dev
|
||||
- name: Build veloxd
|
||||
run: cmake --build --preset dev --target veloxd
|
||||
- name: Nightly integration run
|
||||
run: python3 tests/integration/nightly_run.py --veloxd build/dev/bin/veloxd --tasks 50 --timeout 180
|
||||
|
||||
gui-dod:
|
||||
# Per-PR GUI M1 DoD gates (gui/docs/pkg-qa-requests-m1.md R3): scroll-60fps and
|
||||
# unhappy-path. The 10-minute rss-flat gate is gui-dod-nightly, not here. GUI's
|
||||
# harness defaults QT_QPA_PLATFORM=offscreen itself, so no Xvfb/compositor needed.
|
||||
# See tests/integration/README.md#gui-m1-definition-of-done-gates-r3 for what each
|
||||
# gate catches and the forced-failure transcript proving it isn't vacuous.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Bootstrap toolchain
|
||||
run: sudo ./tools/bootstrap.sh
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22' # tools/mockd
|
||||
- name: Configure + build
|
||||
run: |
|
||||
cmake --preset dev
|
||||
cmake --build --preset dev --target gui-dod-harness
|
||||
- name: Install mockd
|
||||
run: cd tools/mockd && npm ci
|
||||
- name: Gates
|
||||
run: |
|
||||
gui/tests/dod/run.sh scroll-60fps --json scroll.json
|
||||
gui/tests/dod/run.sh unhappy-path --json unhappy.json
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: gui-dod-${{ github.run_id }}
|
||||
path: "*.json"
|
||||
|
||||
gui-dod-nightly:
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Bootstrap toolchain
|
||||
run: sudo ./tools/bootstrap.sh
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Configure + build
|
||||
run: |
|
||||
cmake --preset dev
|
||||
cmake --build --preset dev --target gui-dod-harness
|
||||
- run: cd tools/mockd && npm ci
|
||||
- name: RSS soak (10 min)
|
||||
run: gui/tests/dod/run.sh rss-flat --json rss.json
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: gui-dod-rss-${{ github.run_id }}
|
||||
path: rss.json
|
||||
|
||||
+14
@@ -45,3 +45,17 @@ crash-*
|
||||
oom-*
|
||||
leak-*
|
||||
timeout-*
|
||||
|
||||
# dpkg-buildpackage output — debhelper's build tree and the produced .deb/.changes, plus
|
||||
# the classic out-of-tree autotools-style build dir some debhelper versions still create.
|
||||
# debian/control, changelog, copyright, rules, postinst, postrm, source/format and
|
||||
# *.lintian-overrides are real packaging source and stay tracked; everything below is
|
||||
# regenerated by `dpkg-buildpackage` on every run.
|
||||
debian/.debhelper/
|
||||
debian/velox/
|
||||
debian/files
|
||||
debian/*.substvars
|
||||
debian/*.debhelper.log
|
||||
debian/*.debhelper
|
||||
debian/debhelper-build-stamp
|
||||
obj-*/
|
||||
|
||||
@@ -12,6 +12,7 @@ Read this before touching anything. Then read your lane brief in [docs/agents/](
|
||||
| GUI | `gui/` | everything else |
|
||||
| EXT | `extension/` | everything else |
|
||||
| PKG/QA | `packaging/`, `.github/`, `tools/testserver/`, `tests/integration/`, `tests/e2e/`, root build files | any lane's feature code |
|
||||
| PORT | `**/platform/<os>/**`, `packaging/macos/`, `packaging/windows/` | all logic, and `cmake/` — see [ADR 0020](docs/adr/0020-cross-platform-strategy.md) |
|
||||
|
||||
If your task seems to require editing another lane's files, that is a signal the interface
|
||||
is wrong. **File the request; don't reach across.**
|
||||
|
||||
@@ -29,6 +29,10 @@ option(VELOX_BUILD_FUZZ "Build libFuzzer targets (clang only)" OFF)
|
||||
option(VELOX_ENABLE_MEDIA "Build the HLS/DASH media grabber (M4)" OFF)
|
||||
option(VELOX_WERROR "Treat warnings as errors" ON)
|
||||
|
||||
# Central OS detection (docs/adr/0020-cross-platform-strategy.md). Included early: the
|
||||
# dependency finds below gate Linux-only libraries on VELOX_OS_LINUX.
|
||||
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/platform.cmake)
|
||||
|
||||
# Project-wide warning flags. Set via add_compile_options (a directory property), not the
|
||||
# CMAKE_CXX_FLAGS cache variable, because the dev/tsan presets overwrite that cache var
|
||||
# wholesale for sanitizer flags — a target's warnings must not ride on it.
|
||||
@@ -55,12 +59,24 @@ endif()
|
||||
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/daemon/CMakeLists.txt)
|
||||
find_package(SQLite3 REQUIRED)
|
||||
find_package(nlohmann_json 3.11 REQUIRED)
|
||||
# libsecret / Secret Service is Linux-only (docs/adr/0020-cross-platform-strategy.md,
|
||||
# docs/08-porting.md): macOS uses Keychain behind the same credential-store seam, so
|
||||
# this stays REQUIRED on Linux and simply absent elsewhere — PORT's job is to add the
|
||||
# macOS side of that seam, not to touch this find.
|
||||
if(VELOX_OS_LINUX)
|
||||
find_package(PkgConfig REQUIRED)
|
||||
pkg_check_modules(LIBSECRET REQUIRED IMPORTED_TARGET libsecret-1)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(VELOX_BUILD_GUI AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/gui/CMakeLists.txt)
|
||||
# DBus (gui/docs/pkg-qa-requests-m1.md R4) backs the org.freedesktop.portal
|
||||
# GlobalShortcuts path — Linux/portal-only, same reasoning as libsecret above.
|
||||
if(VELOX_OS_LINUX)
|
||||
find_package(Qt6 6.6 REQUIRED COMPONENTS Widgets Svg Network DBus LinguistTools)
|
||||
else()
|
||||
find_package(Qt6 6.6 REQUIRED COMPONENTS Widgets Svg Network LinguistTools)
|
||||
endif()
|
||||
qt_standard_project_setup()
|
||||
endif()
|
||||
|
||||
@@ -105,3 +121,63 @@ if(VELOX_BUILD_TESTS)
|
||||
endif()
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
# --- Install rules (M6 packaging; docs/07-packaging.md; owned by PKG/QA) ------------
|
||||
#
|
||||
# Guarded by `if(TARGET ...)`, not by editing each lane's own CMakeLists.txt: install()
|
||||
# is a packaging concern, and reaching into daemon/CMakeLists.txt (DAEMON's), cli's build
|
||||
# file (also DAEMON's) or gui/CMakeLists.txt (GUI's) to add it there would cross a lane
|
||||
# boundary CLAUDE.md draws on purpose. Every target below is defined by its own lane;
|
||||
# this only says where the packaged binary already built by that lane's rules goes.
|
||||
#
|
||||
# libveloxcore is intentionally absent here: it stays a static library linked into each
|
||||
# binary (user decision, see docs/07-packaging.md) — there is no .so to install.
|
||||
#
|
||||
# CMAKE_INSTALL_LIBDIR is multiarch-adjusted (lib/x86_64-linux-gnu/) by GNUInstallDirs on
|
||||
# Debian; systemd user units and the Mozilla native-messaging directory are NOT
|
||||
# architecture-specific paths, so those two destinations are written literally
|
||||
# (lib/systemd/user, lib/mozilla/...) rather than built from that variable.
|
||||
include(GNUInstallDirs)
|
||||
|
||||
if(TARGET veloxd)
|
||||
install(TARGETS veloxd RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
|
||||
install(FILES ${CMAKE_SOURCE_DIR}/packaging/man/veloxd.8
|
||||
DESTINATION ${CMAKE_INSTALL_MANDIR}/man8)
|
||||
# velox.socket ships alongside velox.service: DAEMON wired real sd_listen_fds()
|
||||
# socket activation (daemon/src/rpc/systemd_activation.cpp) — see
|
||||
# packaging/systemd/README.md for the pair's own rationale.
|
||||
install(FILES ${CMAKE_SOURCE_DIR}/packaging/systemd/velox.service
|
||||
${CMAKE_SOURCE_DIR}/packaging/systemd/velox.socket
|
||||
DESTINATION lib/systemd/user)
|
||||
endif()
|
||||
|
||||
if(TARGET velox)
|
||||
install(TARGETS velox RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
|
||||
# DAEMON owns cli/ and its man page (CLAUDE.md); packaging only installs it.
|
||||
install(FILES ${CMAKE_SOURCE_DIR}/cli/man/velox.1
|
||||
DESTINATION ${CMAKE_INSTALL_MANDIR}/man1)
|
||||
endif()
|
||||
|
||||
if(TARGET velox-gui)
|
||||
install(TARGETS velox-gui RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
|
||||
install(FILES ${CMAKE_SOURCE_DIR}/packaging/man/velox-gui.1
|
||||
DESTINATION ${CMAKE_INSTALL_MANDIR}/man1)
|
||||
install(FILES ${CMAKE_SOURCE_DIR}/packaging/desktop/velox.desktop
|
||||
DESTINATION ${CMAKE_INSTALL_DATADIR}/applications)
|
||||
foreach(iconsize 16 22 24 32 48 64 128 256)
|
||||
install(FILES
|
||||
${CMAKE_SOURCE_DIR}/packaging/icons/hicolor/${iconsize}x${iconsize}/apps/velox.png
|
||||
DESTINATION ${CMAKE_INSTALL_DATADIR}/icons/hicolor/${iconsize}x${iconsize}/apps)
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
# nmhost and its manifest are DAEMON's (CLAUDE.md: nmhost/ and packaging/nativehost/).
|
||||
# This lights up the moment DAEMON's target exists, the same way the add_subdirectory()
|
||||
# guards above do — no coordinated edit needed when it lands.
|
||||
if(TARGET velox-nmhost)
|
||||
install(TARGETS velox-nmhost RUNTIME DESTINATION libexec/velox)
|
||||
install(FILES ${CMAKE_SOURCE_DIR}/packaging/nativehost/com.velox.host.json
|
||||
DESTINATION lib/mozilla/native-messaging-hosts)
|
||||
else()
|
||||
message(STATUS "velox: nmhost has not landed yet — native-messaging manifest not installed.")
|
||||
endif()
|
||||
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
.TH VELOX 1 "2026-09-12" "Velox" "User Commands"
|
||||
.SH NAME
|
||||
velox \- command-line client for veloxd, the Velox download manager daemon
|
||||
.SH SYNOPSIS
|
||||
.B velox
|
||||
.I command
|
||||
.RI [ options ]
|
||||
.SH DESCRIPTION
|
||||
.B velox
|
||||
talks to
|
||||
\fBveloxd\fR
|
||||
over its Unix-domain socket and gives a scriptable, terminal-first view of the same
|
||||
downloads the GUI and the Firefox extension see. It does no downloading itself \(em every
|
||||
command is a thin JSON-RPC call; the daemon owns all state.
|
||||
.PP
|
||||
.B veloxd
|
||||
must already be running (see
|
||||
.B ENVIRONMENT
|
||||
below for how
|
||||
.B velox
|
||||
finds it, and
|
||||
.BR systemctl (1)
|
||||
.RI ( "systemctl --user status velox" )
|
||||
for whether it's up). If nothing is listening on the socket,
|
||||
.B velox
|
||||
exits with status 3 rather than hanging.
|
||||
.SH COMMANDS
|
||||
.TP
|
||||
.BI "add " url " [\-\-dir " dir "] [\-\-out " name "] [\-\-segments " n ]
|
||||
Add a new download. Prints the new task's id and initial state.
|
||||
.RS
|
||||
.TP
|
||||
.BI "\-\-dir " dir
|
||||
Destination directory. Must resolve inside one of the daemon's configured
|
||||
.B saveTo.allowedRoots
|
||||
or the call fails; omit to use the configured default download directory.
|
||||
.TP
|
||||
.BI "\-\-out " name
|
||||
Filename to save as. Omit to derive one from the URL (or, once the daemon has probed it,
|
||||
from the server's own
|
||||
.BR Content-Disposition ).
|
||||
.TP
|
||||
.BI "\-\-segments " n
|
||||
Requested connection count for this download, 1\(en32. The daemon may use fewer \(em a
|
||||
per-host cap or a source that turns out not to support resuming both lower it. The
|
||||
.B ls
|
||||
table (and
|
||||
.RI "\-\-json's " segments
|
||||
field) show what was actually granted, not what was asked for.
|
||||
.RE
|
||||
.TP
|
||||
.B ls
|
||||
List every download the daemon knows about: id, state, progress, and filename. With
|
||||
.BR \-\-json ", the raw " download.list " result instead of the table."
|
||||
.TP
|
||||
.BI "pause " id " [" id " ...]"
|
||||
Pause one or more downloads by id. A download already paused, or already finished, is
|
||||
left alone \(em not an error.
|
||||
.TP
|
||||
.BI "resume " id " [" id " ...]"
|
||||
Resume one or more paused downloads.
|
||||
.TP
|
||||
.BI "rm " id " [" id " ...] " "[\-\-delete\-file]"
|
||||
Remove one or more downloads from the list. Without
|
||||
.BR \-\-delete\-file ,
|
||||
any partial data on disk
|
||||
.RI ( .veloxpart / .veloxpart.meta )
|
||||
is discarded but a
|
||||
.B completed
|
||||
file is left in place. With
|
||||
.BR \-\-delete\-file ,
|
||||
the finished file is deleted too \(em there is deliberately no default for this flag; you
|
||||
must say which you mean every time.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.B \-\-json
|
||||
Print the raw JSON-RPC result instead of a formatted table. Works with every command;
|
||||
combine with
|
||||
.BR jq (1)
|
||||
for scripting. On error, the JSON form is an
|
||||
.B {"error": {...}}
|
||||
object on stdout rather than a message on stderr.
|
||||
.TP
|
||||
.B \-h ", " \-\-help
|
||||
Print usage and exit 0.
|
||||
.SH EXIT STATUS
|
||||
.TP
|
||||
.B 0
|
||||
Success.
|
||||
.TP
|
||||
.B 1
|
||||
The daemon reached the call but returned a JSON-RPC error (bad task id, path outside the
|
||||
allowed roots, and so on). The message is on stderr, or in the JSON error object with
|
||||
.BR \-\-json .
|
||||
.TP
|
||||
.B 2
|
||||
Usage error \(em missing argument, unknown option, or unknown command.
|
||||
.TP
|
||||
.B 3
|
||||
Could not reach
|
||||
.B veloxd
|
||||
at all: not running, or its socket is missing or unreachable.
|
||||
.SH ENVIRONMENT
|
||||
.TP
|
||||
.B XDG_RUNTIME_DIR
|
||||
.B velox
|
||||
connects to
|
||||
.IR "$XDG_RUNTIME_DIR/velox/velox.sock" .
|
||||
If unset, it falls back to
|
||||
.IR /run/user/ <uid> ,
|
||||
matching
|
||||
\fBveloxd\fR's
|
||||
own resolution \(em the two must agree for the client to find the daemon, so this is
|
||||
normally left to the desktop session's default rather than set by hand.
|
||||
.SH FILES
|
||||
.TP
|
||||
.I $XDG_RUNTIME_DIR/velox/velox.sock
|
||||
The daemon's Unix-domain socket, mode 0600, same-UID only (\fBSO_PEERCRED\fR checked on
|
||||
every connection \(em this is the transport's authorization, not an extra login).
|
||||
.SH SEE ALSO
|
||||
.BR systemctl (1),
|
||||
.BR jq (1)
|
||||
.PP
|
||||
.I docs/01-architecture.md
|
||||
and
|
||||
.I docs/agents/AGENT-DAEMON.md
|
||||
in the Velox source tree for the daemon's own build order and the wire protocol
|
||||
.B velox
|
||||
speaks.
|
||||
@@ -11,6 +11,7 @@
|
||||
#include "client.hpp"
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_loop.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/uds_server.hpp"
|
||||
#include "store/migrations.hpp"
|
||||
#include "store/settings.hpp"
|
||||
@@ -59,8 +60,9 @@ void run() {
|
||||
CHECK(settings.set_raw("saveTo.defaultDir",
|
||||
"\"" + g_allowed_root + "\"").has_value());
|
||||
}
|
||||
rpc::VeloxDispatcher dispatcher(*db);
|
||||
rpc::UdsServer server(loop, dispatcher, server_sock);
|
||||
rpc::EventHub hub;
|
||||
rpc::VeloxDispatcher dispatcher(*db, hub);
|
||||
rpc::UdsServer server(loop, dispatcher, hub, server_sock);
|
||||
const auto ec = server.start();
|
||||
CHECK(!ec);
|
||||
if (ec) return;
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# cmake/platform.cmake — central OS detection and per-OS source selection.
|
||||
#
|
||||
# Owned by PKG/QA (docs/adr/0020-cross-platform-strategy.md §4-5, amended: PORT owns
|
||||
# **/platform/<os>/** and the macOS/Windows packaging trees, but this file is root build
|
||||
# infrastructure and lands in Phase 0 — CORE's and DAEMON's own seams need
|
||||
# velox_platform_sources() to select their platform/linux/*.cpp before PORT ever starts).
|
||||
#
|
||||
# Sets exactly one of VELOX_OS_LINUX / VELOX_OS_MACOS / VELOX_OS_WINDOWS. Selection lives
|
||||
# here and nowhere else — no lane's own CMakeLists.txt should reimplement this check.
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
||||
set(VELOX_OS_LINUX TRUE)
|
||||
elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
|
||||
set(VELOX_OS_MACOS TRUE)
|
||||
elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows")
|
||||
set(VELOX_OS_WINDOWS TRUE)
|
||||
else()
|
||||
message(FATAL_ERROR "velox: unsupported CMAKE_SYSTEM_NAME '${CMAKE_SYSTEM_NAME}' — "
|
||||
"expected Linux, Darwin or Windows.")
|
||||
endif()
|
||||
|
||||
# velox_platform_sources(<target> <dir>) adds <dir>/platform/<os>/*.cpp to <target>, where
|
||||
# <os> is linux, macos or windows to match this file's VELOX_OS_* selection. <dir> is
|
||||
# relative to the calling lane's own CMakeLists.txt (e.g. src/io, src/rpc) — the seam
|
||||
# headers themselves (<dir>/platform/*.hpp) are not globbed here, they're ordinary sources
|
||||
# the owning lane already lists.
|
||||
function(velox_platform_sources target dir)
|
||||
if(VELOX_OS_LINUX)
|
||||
set(os_dir "linux")
|
||||
elseif(VELOX_OS_MACOS)
|
||||
set(os_dir "macos")
|
||||
elseif(VELOX_OS_WINDOWS)
|
||||
set(os_dir "windows")
|
||||
endif()
|
||||
|
||||
file(GLOB platform_sources CONFIGURE_DEPENDS
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/${dir}/platform/${os_dir}/*.cpp")
|
||||
|
||||
if(NOT platform_sources)
|
||||
message(WARNING "velox: velox_platform_sources(${target} ${dir}) found no "
|
||||
"sources under ${dir}/platform/${os_dir}/ — is the seam missing "
|
||||
"its ${os_dir} backend?")
|
||||
endif()
|
||||
|
||||
target_sources(${target} PRIVATE ${platform_sources})
|
||||
endfunction()
|
||||
@@ -506,7 +506,12 @@ def emit_field_parse(f: Field, indent: str) -> list[str]:
|
||||
f'{i} const auto it = j.find("{f.name}");']
|
||||
if f.optional:
|
||||
# Absent and null mean the same thing: the field is not set. A client that omits
|
||||
# a nullable field and one that sends null are treated identically on purpose.
|
||||
# a nullable field and one that sends null are treated identically on purpose --
|
||||
# correct for create-style params, where there is no existing value to distinguish
|
||||
# "never set" from "explicitly cleared". Patch-style fields need the distinction
|
||||
# (download.update's patch: "an explicit null clears a nullable field") and get an
|
||||
# opt-in exception via x-clearable per ADR 0018 (not implemented yet: this is the
|
||||
# decision record, not the generator change).
|
||||
o.append(f"{i} if (it != j.end() && !it->is_null()) {{")
|
||||
o += emit_value_parse(f.type, "(*it)", "val", "fp", i + " ")
|
||||
o.append(f"{i} out.{m} = std::move(val);")
|
||||
|
||||
@@ -21,7 +21,7 @@ fixtures/
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"name": "download.add — start an ISO now, into the Programs category",
|
||||
"name": "download.add — add an ISO for later, into the Programs category",
|
||||
"description": "Why this case is worth pinning.",
|
||||
"transport": "uds", // optional: replay only on this transport
|
||||
"requires": "...", // optional: a condition a plain server cannot produce
|
||||
@@ -81,3 +81,27 @@ cases in `tests/integration/`.
|
||||
correct response is *no response*: past 750 ms the extension must abandon the offer and let
|
||||
Firefox download normally. A download manager that eats downloads when its daemon is down
|
||||
is worse than no download manager.
|
||||
|
||||
## No fixture may pair a real external URL with `startMode: "now"`
|
||||
|
||||
This suite replays every fixture against a real, live `veloxd` (`tests/conformance/run.sh`),
|
||||
not just `mockd`. `mockd` never actually fetches anything, so it hid this for a while: a
|
||||
fixture with `startMode: "now"` (or `"queue"` into a running queue — anything that gets
|
||||
admitted to the scheduler right away) and a real, resolvable URL makes a **real** daemon
|
||||
actually start downloading it, for real, onto whatever machine runs the suite. This
|
||||
happened — twice, with `download.add.json` pointed at a ~6 GB Ubuntu ISO, straight into the
|
||||
developer's real `~/Downloads`.
|
||||
|
||||
The fix in each case is one of:
|
||||
- `startMode: "later"` — exercises the add path (validation, category assignment, the
|
||||
event) without ever handing the task to the engine;
|
||||
- a URL under `example.org`/`example.com` (IANA-reserved for exactly this, RFC 2606) —
|
||||
resolvable enough to validate as a URL, never a real download source;
|
||||
- `requires`, if the fixture's entire point needs a real transfer to fail in a specific way
|
||||
(see `errors/download.add.disk-full.json`) — skipped by default, so it only ever runs
|
||||
where the condition has actually been arranged.
|
||||
|
||||
A real `saveDir` gets the same treatment for the same reason: an absolute path like
|
||||
`/home/sami/Downloads/...` only means anything on the machine that fixture was written on.
|
||||
Omit `saveDir` and let `saveTo.defaultDir` apply, or use a relative-feeling path under a
|
||||
root the runner controls.
|
||||
|
||||
@@ -1,22 +1,23 @@
|
||||
{
|
||||
"name": "capture.offer — attachment on a monitored type is taken",
|
||||
"description": "Golden fixture. tests/conformance replays this against the real daemon AND the TS client. If either side drifts, this goes red before the lanes ever integrate.",
|
||||
"description": "Golden fixture. tests/conformance replays this against the real daemon AND the TS client. If either side drifts, this goes red before the lanes ever integrate. url is example.org (RFC 2606), not a real download source: 'take' against a real veloxd (tests/conformance/run.sh) admits a real task and hands it to the engine for real, and no fixture may do that against a real external URL. contentLength is a plausible-but-small 5 MiB rather than a real ISO's size: the 'Programs' category's saveDir is a migration-seeded builtin (~/Downloads/Programs, daemon/src/store/migrations/0001_initial.sql), not something an isolated test run's settings can redirect, so 'take' always sparse-preallocates into that real path on whatever machine runs this suite -- keeping the declared size small keeps that footprint trivial instead of a real ISO's worth of disk. transport is uds only: a real 'take' persists an active task, so replaying this same fixture again on a second live transport against the same daemon would correctly dedupe against it (capture.offer dedupes by exact URL) and get 'ignore' instead -- an artifact of replaying one fixture against one shared daemon over two transports, not a behaviour to golden.",
|
||||
"transport": "uds",
|
||||
"request": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 42,
|
||||
"method": "capture.offer",
|
||||
"params": {
|
||||
"url": "https://releases.ubuntu.com/26.04/ubuntu-26.04-desktop-amd64.iso",
|
||||
"url": "https://example.org/dl/ubuntu-26.04-desktop-amd64.iso",
|
||||
"method": "GET",
|
||||
"tabUrl": "https://releases.ubuntu.com/26.04/",
|
||||
"tabUrl": "https://example.org/26.04/",
|
||||
"headers": {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:154.0) Gecko/20100101 Firefox/154.0",
|
||||
"Referer": "https://releases.ubuntu.com/26.04/",
|
||||
"Referer": "https://example.org/26.04/",
|
||||
"Accept": "*/*"
|
||||
},
|
||||
"cookies": [],
|
||||
"contentType": "application/octet-stream",
|
||||
"contentLength": 6228541440,
|
||||
"contentLength": 5242880,
|
||||
"contentDisposition": "attachment; filename=\"ubuntu-26.04-desktop-amd64.iso\"",
|
||||
"filename": "ubuntu-26.04-desktop-amd64.iso",
|
||||
"origin": "moz-extension://11111111-2222-3333-4444-555555555555"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "download.add \u2014 start an ISO now, into the Programs category",
|
||||
"description": "The ordinary add path. saveDir is canonicalized and checked against the allowed roots before anything is written.",
|
||||
"name": "download.add — add an ISO for later, into the Programs category",
|
||||
"description": "The ordinary add path. saveDir is canonicalized and checked against the allowed roots before anything is written. startMode is 'later' deliberately: this suite replays against a real veloxd (tests/conformance/run.sh), and a real daemon given startMode 'now' would actually start fetching url for real. No fixture may pair a real external URL with startMode 'now' -- see contracts/fixtures/README.md.",
|
||||
"request": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 11,
|
||||
@@ -8,10 +8,9 @@
|
||||
"params": {
|
||||
"url": "https://releases.ubuntu.com/26.04/ubuntu-26.04-desktop-amd64.iso",
|
||||
"filename": "ubuntu-26.04-desktop-amd64.iso",
|
||||
"saveDir": "/home/sami/Downloads/Programs",
|
||||
"categoryId": "programs",
|
||||
"segments": 8,
|
||||
"startMode": "now"
|
||||
"startMode": "later"
|
||||
}
|
||||
},
|
||||
"response": {
|
||||
@@ -19,13 +18,13 @@
|
||||
"id": 11,
|
||||
"result": {
|
||||
"taskId": "$uuid",
|
||||
"state": "connecting",
|
||||
"state": "paused",
|
||||
"duplicate": null
|
||||
}
|
||||
},
|
||||
"assertions": [
|
||||
"the .veloxpart file is created sparse and preallocated at the final size",
|
||||
"saveDir resolves inside saveTo.allowedRoots, or the call fails -32011 having written nothing",
|
||||
"saveDir is omitted here on purpose: it resolves to saveTo.defaultDir, which is itself checked against saveTo.allowedRoots the same way an explicit saveDir would be -- see errors/download.add.invalid-path.json for the -32011 case",
|
||||
"startMode 'later' lands the task in 'paused' and never hands it to the engine, so nothing is fetched and no .veloxpart is created yet -- that only happens once the task is actually started (download.start.json, or startMode 'now'/'queue' against a source this suite controls)",
|
||||
"event.task.added is emitted to every subscriber before this reply is sent"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
],
|
||||
"defaults": {
|
||||
"url": "https://example.org/",
|
||||
"categoryId": "compressed",
|
||||
"categoryId": "programs",
|
||||
"startMode": "queue",
|
||||
"queueId": "main"
|
||||
}
|
||||
|
||||
@@ -30,5 +30,6 @@
|
||||
"the version check is transport-independent; this is replayed on the Unix socket so it is not masked by -32002",
|
||||
"data.expected is the daemon's own current protocol version string (kProtocolVersion), not a bare major and not pinnable in a golden file -- the conformance compare on error payloads is on `code` only, structural elsewhere, so echoing the live version is fine"
|
||||
],
|
||||
"transport": "uds"
|
||||
"transport": "uds",
|
||||
"closesConnection": true
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "limiter.get \u2014 the limiter is off",
|
||||
"description": "globalBps still carries the last configured value so the GUI can restore it when the user re-enables the limit.",
|
||||
"description": "globalBps still carries the last configured value so the GUI can restore it when the user re-enables the limit. applyToRunning is a write-only instruction on limiter.set (\"retune already-running transfers now\", not a persisted setting), so it never comes back from get.",
|
||||
"request": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 52,
|
||||
@@ -12,11 +12,11 @@
|
||||
"id": 52,
|
||||
"result": {
|
||||
"enabled": false,
|
||||
"globalBps": 2097152,
|
||||
"applyToRunning": false
|
||||
"globalBps": 2097152
|
||||
}
|
||||
},
|
||||
"assertions": [
|
||||
"enabled false means no throttling regardless of globalBps"
|
||||
"enabled false means no throttling regardless of globalBps",
|
||||
"applyToRunning is absent, not false: it's meaningless outside a limiter.set call"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -26,9 +26,17 @@ add_library(veloxcore STATIC
|
||||
src/task/digest.cpp
|
||||
src/task/download_task.cpp
|
||||
src/engine.cpp
|
||||
src/rules/filename.cpp
|
||||
src/rules/collision.cpp
|
||||
src/rules/match.cpp
|
||||
)
|
||||
add_library(velox::core ALIAS veloxcore)
|
||||
|
||||
# Per-OS backend behind src/io/platform/*.hpp (docs/adr/0020, docs/08-porting.md). Only
|
||||
# the Linux backend exists (Phase 0); source selection lives in cmake/platform.cmake and
|
||||
# nowhere else, so PORT's platform/macos/ addition needs no edit here.
|
||||
velox_platform_sources(veloxcore src/io)
|
||||
|
||||
target_include_directories(veloxcore
|
||||
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
|
||||
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src # net/*.cpp -> "net/curl_error.hpp"
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
# M7 performance baseline
|
||||
|
||||
Measured against `docs/04-engine-design.md` §8's targets, via `tools/bench/vdm_bench`
|
||||
(see that file's header comment for the exact commands — reproduced below with their
|
||||
actual output). `--preset release`, this machine, 2026-09-11 through 2026-09-12. This is
|
||||
a baseline record, not a sign-off: one of the three numbers below is loopback-only rather
|
||||
than measured against a real 1 Gbit link, stated plainly rather than rounded away — see
|
||||
"Open gaps". The RSS number *did* fail the DoD line in the first pass through this file;
|
||||
it's since been root-caused and fixed (`docs/adr/0017`), not just re-measured.
|
||||
|
||||
## Commands and results
|
||||
|
||||
```
|
||||
$ cmake --preset release && cmake --build --preset release
|
||||
|
||||
$ bin/vdm_bench throughput --size 5G --require-mbps 940 --max-cpu-pct 8
|
||||
throughput: 5368709120 bytes in 2.99s
|
||||
throughput 14371.00 Mbps
|
||||
cpu 196.52 % of one core
|
||||
peak RSS 22.81 MiB
|
||||
```
|
||||
Against `tools/bench/support/local_server.hpp`'s busybox loopback server, not a real 1
|
||||
Gbit link — no such link was available to test against in this environment, so
|
||||
`--require-mbps`/`--max-cpu-pct` weren't meaningfully exercised here (loopback trivially
|
||||
clears 940 Mbps; the 196% CPU figure reflects driving a link far faster than 1 Gbit, not
|
||||
the 1-Gbit-saturated cost the target is about). This needs re-running against a real
|
||||
1 Gbit peer before it can stand as the actual M1/M7 sign-off number.
|
||||
|
||||
```
|
||||
$ bin/vdm_bench heap-profile --tasks 20 --task-size 4M --top 5
|
||||
segment budget: 32 live across tasks, engine reports total=32 active=32 starved=9
|
||||
heap-profile: peak RSS 45.41 MiB, 20 tasks, 8 segments assumed
|
||||
```
|
||||
Default `Config` (`default_segments=8`, `max_active_segments=32`, `default_buffer_bytes=1
|
||||
MiB`), default `--task-size 4M`. **Now clears the 60 MB line** (45.41 MiB), after the real
|
||||
fix below — root-caused, not just re-measured. Superseded the original `load` run's ~70
|
||||
MiB number quoted in earlier drafts of this file; see `docs/adr/0017`.
|
||||
|
||||
```
|
||||
$ bin/vdm_bench alloc-check --size 512M --window-s 2
|
||||
alloc-check: 4 allocations in 2.00s (budget 15)
|
||||
```
|
||||
Clears the no-allocation-on-the-hot-path bar (docs/agents/AGENT-CORE.md) comfortably.
|
||||
This number is *after* a real fix landed in the same change:
|
||||
`net::HttpClient::Impl::drain_commands` was constructing an (always-allocating, in
|
||||
libstdc++) `std::deque` on every worker-loop iteration regardless of whether any command
|
||||
was actually pending — once per curl_multi_poll wake, i.e. on the transfer hot path. Fixed
|
||||
by checking `w.queue.empty()` under the lock before touching `local` at all. Before the
|
||||
fix this bench reported thousands of allocations/sec under any sustained transfer.
|
||||
|
||||
## ASan / UBSan / TSan (M1 DoD: "20-task load test... clean")
|
||||
|
||||
- `--preset dev` (ASan+UBSan) and `--preset tsan`: the full `core/` test suite (40 ctest
|
||||
cases, including `veloxcore_engine_test`'s hostile-mode suite and `veloxcore_budget_test`)
|
||||
and all three `tools/bench` smoke tests pass clean on both presets.
|
||||
- Two real bugs were caught and fixed getting here:
|
||||
- `DownloadTaskState::quiesce()` (engine shutdown / `Engine`'s destructor) cleared the
|
||||
`workers` map synchronously right after issuing an async `transfer.cancel()`, racing
|
||||
the HttpClient worker thread's still-in-flight write callback into a heap-use-after-free
|
||||
on the segment's ring buffer — ASan-caught via `alloc-check`, which (by design) drops
|
||||
its `Engine` while a download is still active. Fixed by having `quiesce()` wait for
|
||||
each worker to drain itself through the same `seg_finished` path every other exit uses,
|
||||
instead of tearing the map down itself.
|
||||
- `SegWorker::speed_bps` (the polled-progress fix, see `engine_polled_progress_reports_
|
||||
nonzero_speed`) was written only by a segment's own curl callback and, before this
|
||||
session, only ever read from that same thread (`emit_progress_if_due`, called from the
|
||||
same callback) — safe without synchronization. Reading it from `snapshot_progress()`
|
||||
(any thread calling `DownloadHandle::progress()`) broke that invariant: `workers_mu`'s
|
||||
shared_lock protects the `workers` map's structure, not an individual `SegWorker`'s
|
||||
mutable fields. TSan-caught. Fixed with `std::atomic<double>` (relaxed: this is an
|
||||
informational EMA, nothing synchronizes real state on it) rather than adding a lock to
|
||||
the write side.
|
||||
- The `tools/bench load` ctest registration still runs at reduced concurrency
|
||||
(`--tasks 8 --segments 2`) under sanitizer presets (`tools/bench/CMakeLists.txt`) from
|
||||
when this was written against `docs/adr/0016`'s postscript — see `docs/adr/0017`'s "Open
|
||||
gaps" note: that straggler is now suspected to have been the *same* root cause as the RSS
|
||||
bug, not re-verified at the DoD's full shape under `--preset tsan` in this change.
|
||||
|
||||
## Open gaps
|
||||
|
||||
1. ~~RSS is ~70 MB against a 60 MB target~~ **Fixed — see `docs/adr/0017`.** Root cause
|
||||
was not, as first guessed here, an `ADR 0012` arithmetic gap or per-queued-handle curl
|
||||
overhead: `SegmentBudget::confirm_slot()` only checked a task's *own* target against
|
||||
its own held count, never the engine-wide `active_` sum, so it could (and under real
|
||||
20-task/8-segment contention, reliably did) admit segments well past
|
||||
`max_active_segments` — `heap-profile` caught it directly: `budget.active` reading
|
||||
56–86 against a `total` of 32. Fixed at the budget level (the one place that can
|
||||
actually enforce the invariant); `docs/adr/0012`'s own arithmetic was fine all along.
|
||||
2. **Throughput/CPU numbers are loopback-only.** No 1 Gbit link was available to test
|
||||
against; re-run `throughput --size 5G --require-mbps 940 --max-cpu-pct 8` against a real
|
||||
one before treating this as signed off.
|
||||
3. **`docs/adr/0016`**: `rate::RateLimiter`'s global-limit path (byte-rate pacing, a
|
||||
different subsystem from the segment-admission bug in `docs/adr/0017`) has no fairness
|
||||
ordering under heavy segment contention (a shared `TokenBucket`'s peek/commit race can
|
||||
starve a waiter indefinitely) — a real, separate, still-open gap for the "global
|
||||
bandwidth cap with many concurrent downloads" scenario.
|
||||
4. **The TSan-only load-test straggler** noted in `docs/adr/0016`'s postscript, found
|
||||
before `docs/adr/0017`'s fix landed: plausibly the *same* root cause (a segment denied
|
||||
admission with nothing to wake it, worse under TSan's slowdown widening the window a
|
||||
deferred yield can sit in) rather than the `CURLOPT_LOW_SPEED_TIME` guess that ADR
|
||||
originally offered — not reverified at the DoD's full 20-task/8-segment shape under
|
||||
`--preset tsan` in this change (the sanitizer-preset smoke registration still runs at
|
||||
reduced concurrency; see `tools/bench/CMakeLists.txt`). Worth re-running before treating
|
||||
it as closed.
|
||||
@@ -1,12 +1,12 @@
|
||||
# `libveloxcore` — public API
|
||||
|
||||
**Status: M1 in progress.** `util/`, `net/` (http_client, probe, url, content_disposition),
|
||||
`io/` (sparse_file, write_buffer), `meta/veloxpart`, and `segment/` (segmenter, budget)
|
||||
are landed. The **download entry point** — `vdm::Engine`, `vdm::task::DownloadSpec` /
|
||||
`DownloadHandle` / `DownloadCallbacks` — is sketched in `vdm/engine.hpp` and
|
||||
`vdm/task/download.hpp` and **out for DAEMON review**: see
|
||||
[`core/docs/engine-api-m1.md`](../../docs/engine-api-m1.md). Bodies land in CORE stage 8;
|
||||
build against the value types now.
|
||||
`io/` (sparse_file, write_buffer), `meta/veloxpart`, `segment/` (segmenter, budget),
|
||||
`rate/` (token_bucket), `task/`+`engine.hpp` (the download engine itself — `vdm::Engine`,
|
||||
`DownloadSpec`/`DownloadHandle`/`DownloadCallbacks`), and `rules/` (filename sanitization,
|
||||
collision policy, rule-table matching) are landed. `media/` is M4, not started — see
|
||||
[`core/docs/engine-api-m1.md`](../../docs/engine-api-m1.md) for the engine API's own
|
||||
DAEMON-review history.
|
||||
|
||||
Layering (CLAUDE.md §3): this library knows nothing about JSON, SQL, Qt, or RPC. Input is
|
||||
a spec value; output is bytes on disk plus typed callbacks. DAEMON projects engine state
|
||||
@@ -75,3 +75,44 @@ Sink interface — core does no I/O itself. `LogSink` abstract base; DAEMON inst
|
||||
via `set_log_sink()`, default discards. `CallbackSink` adapter (with a min-level filter).
|
||||
`VDM_LOG_{TRACE,DEBUG,INFO,WARN,ERROR}(category, fmt, args...)` — `std::format` syntax,
|
||||
only formatted when a sink is installed and wants the level.
|
||||
|
||||
---
|
||||
|
||||
## `rules/` — filename sanitization, collision policy, rule-table matching
|
||||
|
||||
Pure functions only: no I/O, no filesystem access, no notion of the wire `Rule` type or
|
||||
its JSON/SQL representation. DAEMON owns the rule table (storage, `rules.upsert`, the
|
||||
generated `Rule` type) and decodes it into the plain structs below before calling in.
|
||||
|
||||
### `vdm/rules/filename.hpp`
|
||||
|
||||
`sanitize_filename(raw, max_bytes = 255)` — turns a raw candidate (from
|
||||
`net::parse_content_disposition` or `net::url_filename`, neither of which is
|
||||
filesystem-safe by design — see their own headers) into one safe to create on ext4, APFS,
|
||||
and NTFS alike: strips separators/control bytes, folds NTFS-illegal characters to `_`,
|
||||
neutralizes reserved Windows device names (`CON`, `COM1`, ...), and clamps length on a
|
||||
UTF-8 boundary. Total: never empty, never throws. **Not** the path-traversal security
|
||||
boundary — that's DAEMON's `fs/safepath`, which runs after this and is the one that
|
||||
matters adversarially.
|
||||
|
||||
### `vdm/rules/collision.hpp`
|
||||
|
||||
`resolve_collision(desired, exists, policy, max_attempts = 1000)` — given an existence
|
||||
predicate (DAEMON supplies a real one; tests supply an in-memory set), finds the next free
|
||||
name Explorer/Finder-style (`"name (1).ext"`, `"name (2).ext"`, ...) under
|
||||
`CollisionPolicy::rename`, or returns `desired` unchanged under `::overwrite`. Never
|
||||
fabricates a guaranteed-unique name past `max_attempts` — returns the last candidate tried
|
||||
and leaves "still colliding" for the caller to treat as a real error.
|
||||
|
||||
### `vdm/rules/match.hpp`
|
||||
|
||||
`match_rules(rules, input) -> optional<RuleAction>` — the evaluation half of
|
||||
`contracts/schema/types/Rule.schema.json`: tries rules in ascending `priority` order
|
||||
(ties keep table order), skips disabled rows, returns the first whose every *present*
|
||||
match clause (`extensions`, `mime_types`, `host_pattern`, `url_pattern`,
|
||||
`min_size_bytes`/`max_size_bytes`) is satisfied — an absent clause is not a constraint,
|
||||
and a size clause never matches speculatively when `MatchInput::size_bytes` is still
|
||||
unknown (pre-probe). `std::nullopt` means no rule matched; the caller's own default
|
||||
category applies. `glob_match(pattern, text)` — the `*`/`?` matcher `host_pattern` and
|
||||
`url_pattern` both use, case-insensitive, bounded work even on a pathological
|
||||
all-`*` pattern (iterative, not recursive).
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
// vdm/rules/collision.hpp — when a chosen filename is already taken in the destination
|
||||
// directory, decide what to try next.
|
||||
//
|
||||
// Pure: takes an existence predicate rather than touching a filesystem itself, so it never
|
||||
// races what it's deciding about and stays testable without one. DAEMON (which owns the
|
||||
// actual directory listing / stat calls, downstream of its own fs/safepath gate) supplies
|
||||
// that predicate; a test supplies an in-memory set.
|
||||
//
|
||||
// This header compiles standalone.
|
||||
|
||||
#ifndef VDM_RULES_COLLISION_HPP
|
||||
#define VDM_RULES_COLLISION_HPP
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace vdm::rules {
|
||||
|
||||
enum class CollisionPolicy {
|
||||
rename, // try "name (1).ext", "name (2).ext", ... until one is free
|
||||
overwrite, // return `desired` unchanged — caller intends to replace what's there
|
||||
};
|
||||
|
||||
// Under `CollisionPolicy::rename`: calls `exists(candidate)` first with `desired` itself,
|
||||
// then with "<stem> (1)<ext>", "<stem> (2)<ext>", ... (Explorer/Finder-style, splitting
|
||||
// `desired` on its last '.' the same way `sanitize_filename`'s truncation does), returning
|
||||
// the first candidate for which it returns false. `exists` is never called with anything
|
||||
// but a single leaf name, never a path.
|
||||
//
|
||||
// `max_attempts` bounds a pathological `exists` that always returns true (this function
|
||||
// always returns — it is not fallible): once reached, the last candidate tried is returned
|
||||
// as-is, still possibly colliding. That is deliberately not papered over with a
|
||||
// fabricated-unique name (a timestamp suffix, say) — silently handing back a name nobody
|
||||
// asked for is exactly the kind of thing that turns into a mystery file days later; a
|
||||
// caller that hits the bound should treat it as a real error, not swallow it here.
|
||||
//
|
||||
// Under `CollisionPolicy::overwrite`, `exists` and `max_attempts` are unused — `desired`
|
||||
// is returned unchanged.
|
||||
[[nodiscard]] std::string resolve_collision(std::string_view desired,
|
||||
const std::function<bool(std::string_view)> &exists,
|
||||
CollisionPolicy policy = CollisionPolicy::rename,
|
||||
int max_attempts = 1000);
|
||||
|
||||
} // namespace vdm::rules
|
||||
|
||||
#endif // VDM_RULES_COLLISION_HPP
|
||||
@@ -0,0 +1,49 @@
|
||||
// vdm/rules/filename.hpp — turn a raw, untrusted filename candidate into one safe to
|
||||
// create on a real filesystem, cross-platform.
|
||||
//
|
||||
// This is NOT the path-traversal security boundary — that's DAEMON's fs/safepath (the
|
||||
// process's one canonicalize-and-verify-against-allowed-roots gate; see its own header
|
||||
// comment). This runs earlier and is cooperative, not adversarial-proof on its own: turn
|
||||
// whatever `net::parse_content_disposition` or `net::url_filename` handed back (see their
|
||||
// headers — neither fully sanitizes for the filesystem, by design; this is where that
|
||||
// finishes) into a *reasonable* candidate so an ordinary download doesn't needlessly
|
||||
// collide with a reserved device name, get silently mangled by NTFS, or get truncated
|
||||
// mid-extension by safepath's own leaf check.
|
||||
//
|
||||
// Total on hostile input: never throws, never asserts, never returns empty (falls back to
|
||||
// a generic name). No JSON, no SQL, no Qt, no RPC, no filesystem access (CLAUDE.md §3) —
|
||||
// pure string transformation.
|
||||
//
|
||||
// This header compiles standalone.
|
||||
|
||||
#ifndef VDM_RULES_FILENAME_HPP
|
||||
#define VDM_RULES_FILENAME_HPP
|
||||
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace vdm::rules {
|
||||
|
||||
// Sanitizes `raw` into a single path component safe to create on ext4, APFS, and NTFS
|
||||
// alike:
|
||||
// - strips path separators ('/' always; '\' too — Windows treats it as one) and collapses
|
||||
// any run of '.' that would otherwise still read as a traversal attempt ("..", "...")
|
||||
// down to a single '.', so a stripped-separator name can't reconstitute one
|
||||
// - strips C0 control bytes (incl. NUL) and DEL (0x7F)
|
||||
// - replaces the other NTFS-illegal characters (`< > : " | ? *`) with '_', so a name
|
||||
// that's fine on Linux doesn't silently fail to sync/export to a Windows-formatted
|
||||
// drive or SMB share
|
||||
// - strips trailing '.' and ' ' (both are NTFS traps: silently dropped by the Win32 API,
|
||||
// so "name." and "name" would otherwise collide invisibly on export)
|
||||
// - a reserved Windows device name (CON, PRN, AUX, NUL, COM1–9, LPT1–9), matched
|
||||
// case-insensitively against the part before the first '.' (or the whole name if
|
||||
// there's no '.'), gets a trailing '_' so it stops shadowing a device
|
||||
// - clamps to `max_bytes` (default 255, the common ext4/APFS/NTFS component limit),
|
||||
// cutting on a UTF-8 boundary and preferring to keep a short trailing extension intact
|
||||
// - empty, or entirely stripped down to nothing, falls back to "download"
|
||||
[[nodiscard]] std::string sanitize_filename(std::string_view raw, std::size_t max_bytes = 255);
|
||||
|
||||
} // namespace vdm::rules
|
||||
|
||||
#endif // VDM_RULES_FILENAME_HPP
|
||||
@@ -0,0 +1,91 @@
|
||||
// vdm/rules/match.hpp — the pure evaluation half of the rules engine
|
||||
// (contracts/schema/types/Rule.schema.json): given a rule table and what's known about one
|
||||
// candidate download, decide which rule fires and what it says to do.
|
||||
//
|
||||
// DAEMON owns the rule table itself — storage, `rules.upsert`/`rules.list`, the wire
|
||||
// `Rule` type generated from the contract. It decodes its own JSON/SQL representation into
|
||||
// the plain structs below and calls in; core never sees JSON, SQL, or the generated
|
||||
// protocol types (CLAUDE.md §3) — these structs mirror the contract's shape in CORE's own
|
||||
// vocabulary, the same relationship `util/error.hpp`'s `Error` has to the wire error codes.
|
||||
//
|
||||
// Pure and total: no I/O, no throw, no crash on any input (an empty table, an empty
|
||||
// pattern, a rule with every match clause absent).
|
||||
//
|
||||
// This header compiles standalone.
|
||||
|
||||
#ifndef VDM_RULES_MATCH_HPP
|
||||
#define VDM_RULES_MATCH_HPP
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
namespace vdm::rules {
|
||||
|
||||
enum class StartMode { now, later, queue }; // mirrors contracts' StartMode
|
||||
enum class CaptureVerdict { take, ignore }; // mirrors RuleAction.capture
|
||||
|
||||
// All present clauses must match; an absent clause is not a constraint
|
||||
// (contracts/schema/types/Rule.schema.json's own wording for RuleMatch).
|
||||
struct RuleMatch {
|
||||
std::optional<std::vector<std::string>> extensions; // no leading '.'; matched
|
||||
// case-insensitively
|
||||
std::optional<std::vector<std::string>> mime_types; // matched case-insensitively
|
||||
std::optional<std::string> host_pattern; // glob_match against the host
|
||||
std::optional<std::string> url_pattern; // glob_match against the whole URL
|
||||
std::optional<std::uint64_t> min_size_bytes; // inclusive
|
||||
std::optional<std::uint64_t> max_size_bytes; // inclusive
|
||||
};
|
||||
|
||||
// What to do with a matching download.
|
||||
struct RuleAction {
|
||||
std::optional<std::string> category_id;
|
||||
std::optional<std::string> save_dir;
|
||||
std::optional<std::string> queue_id;
|
||||
std::optional<std::uint32_t> segments;
|
||||
std::optional<StartMode> start_mode;
|
||||
std::optional<CaptureVerdict> capture;
|
||||
};
|
||||
|
||||
// One row of the rules engine.
|
||||
struct Rule {
|
||||
std::string rule_id;
|
||||
bool enabled = true;
|
||||
std::int64_t priority = 0; // lower runs first
|
||||
RuleMatch match;
|
||||
RuleAction action;
|
||||
};
|
||||
|
||||
// What's known about one candidate download, to match rules against. A field being empty
|
||||
// (not `std::nullopt` — these are plain strings, not optionals) means "unknown, matches no
|
||||
// non-empty clause that needs it" — e.g. `size_bytes` is absent pre-probe, so any rule with
|
||||
// a size clause simply doesn't match yet; the caller is expected to re-run matching once
|
||||
// the probe fills it in, same as DownloadSpec itself gets refined post-probe.
|
||||
struct MatchInput {
|
||||
std::string extension; // lowercased, no leading '.'; empty if none
|
||||
std::string mime_type; // lowercased; empty if unknown
|
||||
std::string host; // lowercased effective-URL host
|
||||
std::string url; // the whole effective URL
|
||||
std::optional<std::uint64_t> size_bytes;
|
||||
};
|
||||
|
||||
// Rules are tried in ascending `priority` order (ties broken by table order), skipping
|
||||
// disabled rows; the first whose every present match clause is satisfied wins.
|
||||
// `std::nullopt` means no rule matched — the caller falls back to its own default category
|
||||
// (this function has no notion of a default; that policy lives with the caller).
|
||||
[[nodiscard]] std::optional<RuleAction> match_rules(const std::vector<Rule> &rules,
|
||||
const MatchInput &input);
|
||||
|
||||
// Case-insensitive glob: '*' matches any run of characters including none, '?' matches
|
||||
// exactly one character. No character classes, no escaping — rule patterns are meant to
|
||||
// stay simple (contracts/schema/types/Rule.schema.json's own hostPattern/urlPattern
|
||||
// description gives only "*.example.com" as the example). Exposed on its own because
|
||||
// hostPattern and urlPattern are both just this against different text, and because it has
|
||||
// its own test table worth keeping separate from match_rules's.
|
||||
[[nodiscard]] bool glob_match(std::string_view pattern, std::string_view text) noexcept;
|
||||
|
||||
} // namespace vdm::rules
|
||||
|
||||
#endif // VDM_RULES_MATCH_HPP
|
||||
@@ -100,6 +100,10 @@ class SegmentBudget {
|
||||
std::uint32_t target = 0; // last published
|
||||
SlotTargetFn on_target;
|
||||
std::optional<SteadyTime> starved_since;
|
||||
// confirm_slot() was denied by the engine-wide cap (not by this task's own
|
||||
// target) and hasn't been retried since. Cleared on the task's next successful
|
||||
// confirm_slot(), however that retry was triggered. See release_slot()'s comment.
|
||||
bool waiting_for_slot = false;
|
||||
};
|
||||
|
||||
// A unit of deferred work: callbacks are copied out here so the public entry points
|
||||
@@ -109,7 +113,12 @@ class SegmentBudget {
|
||||
std::optional<std::pair<std::function<void(EngineBudget)>, EngineBudget>> notify_now;
|
||||
};
|
||||
|
||||
[[nodiscard]] std::vector<TaskId> priority_order_locked() const;
|
||||
Plan reallocate_locked();
|
||||
// Appends a retry hint for the highest-priority task with waiting_for_slot set (other
|
||||
// than `exclude`, the task whose release just freed this slot -- see release_slot()'s
|
||||
// comment) to `plan`, if one exists.
|
||||
void wake_one_waiter_locked(Plan &plan, TaskId exclude) const;
|
||||
static void run(Plan &p);
|
||||
[[nodiscard]] std::uint32_t effective_cap_locked(const Task &t) const;
|
||||
[[nodiscard]] EngineBudget snapshot_locked() const;
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
// vdm/io/platform/file_ops.hpp — platform seam for the three Linux-only file-I/O
|
||||
// primitives sparse_file.cpp and meta/veloxpart.cpp use (ADR 0020, docs/08-porting.md).
|
||||
//
|
||||
// One implementation file per OS under platform/<os>/, selected at build time. No #ifdef
|
||||
// here, no OS-specific types in the signatures — a reader of the callers must not need to
|
||||
// know which OS they're on. Linux (platform/linux/file_ops.cpp) is the reference
|
||||
// implementation and is the only one that exists until the PORT lane adds macOS.
|
||||
//
|
||||
// This header compiles standalone.
|
||||
|
||||
#ifndef VDM_IO_PLATFORM_FILE_OPS_HPP
|
||||
#define VDM_IO_PLATFORM_FILE_OPS_HPP
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include "vdm/util/result.hpp"
|
||||
|
||||
namespace vdm::io::platform {
|
||||
|
||||
// Reserve `bytes` for `fd` so a large transfer doesn't fragment or hit ENOSPC mid-write.
|
||||
// Linux: posix_fallocate, falling back to ftruncate on EOPNOTSUPP/ENOSYS/EINVAL exactly as
|
||||
// SparseFile did before this seam existed. Returns whether a true extent-reserving
|
||||
// preallocation happened (false when the ftruncate fallback was used) so callers can keep
|
||||
// reporting SparseFile::preallocated() unchanged.
|
||||
[[nodiscard]] Result<bool> preallocate(int fd, std::uint64_t bytes);
|
||||
|
||||
// Drop cached pages for [offset, offset+len) from the OS page cache, best-effort. A
|
||||
// platform with no equivalent (macOS) is a documented no-op, not a failure — never let
|
||||
// this block or fail the transfer path.
|
||||
void advise_dontneed(int fd, std::uint64_t offset, std::uint64_t len) noexcept;
|
||||
|
||||
// Flush `fd`'s data to durable storage without waiting on metadata that doesn't affect
|
||||
// data readback. Linux: fdatasync.
|
||||
[[nodiscard]] Result<void> flush_durable(int fd);
|
||||
|
||||
} // namespace vdm::io::platform
|
||||
|
||||
#endif // VDM_IO_PLATFORM_FILE_OPS_HPP
|
||||
@@ -0,0 +1,73 @@
|
||||
// vdm/io/platform/linux/file_ops.cpp — Linux backend for the file_ops seam.
|
||||
//
|
||||
// Reference implementation (ADR 0020): moved unchanged from sparse_file.cpp and
|
||||
// meta/veloxpart.cpp. Linux behaviour must never change as a result of a port; if a port
|
||||
// needs a semantic change here, that's an ADR, not a seam edit.
|
||||
|
||||
#include "io/platform/file_ops.hpp"
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cerrno>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
|
||||
namespace vdm::io::platform {
|
||||
namespace {
|
||||
|
||||
Error errno_to_error(int e) noexcept {
|
||||
switch (e) {
|
||||
case ENOSPC:
|
||||
case EDQUOT:
|
||||
return Error::disk_full;
|
||||
case EACCES:
|
||||
case EPERM:
|
||||
case EROFS:
|
||||
return Error::permission_denied;
|
||||
case ENOENT:
|
||||
case ENOTDIR:
|
||||
case EISDIR:
|
||||
case ENAMETOOLONG:
|
||||
case ELOOP:
|
||||
return Error::path_rejected;
|
||||
default:
|
||||
return Error::io_error;
|
||||
}
|
||||
}
|
||||
|
||||
ErrorInfo sys_error(std::string_view what, int e) {
|
||||
return ErrorInfo(errno_to_error(e), std::string(what) + ": " + std::strerror(e));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Result<bool> preallocate(int fd, std::uint64_t bytes) {
|
||||
// posix_fallocate returns the error number directly and does not set errno.
|
||||
int rc = ::posix_fallocate(fd, 0, static_cast<off_t>(bytes));
|
||||
if (rc == 0)
|
||||
return true;
|
||||
if (rc == EOPNOTSUPP || rc == ENOSYS || rc == EINVAL) {
|
||||
if (::ftruncate(fd, static_cast<off_t>(bytes)) != 0)
|
||||
return sys_error("ftruncate", errno);
|
||||
return false;
|
||||
}
|
||||
return sys_error("posix_fallocate", rc);
|
||||
}
|
||||
|
||||
void advise_dontneed(int fd, std::uint64_t offset, std::uint64_t len) noexcept {
|
||||
if (len == 0)
|
||||
return;
|
||||
::posix_fadvise(fd, static_cast<off_t>(offset), static_cast<off_t>(len), POSIX_FADV_DONTNEED);
|
||||
}
|
||||
|
||||
Result<void> flush_durable(int fd) {
|
||||
while (::fdatasync(fd) != 0) {
|
||||
if (errno == EINTR)
|
||||
continue;
|
||||
return sys_error("fdatasync", errno);
|
||||
}
|
||||
return ok();
|
||||
}
|
||||
|
||||
} // namespace vdm::io::platform
|
||||
@@ -9,6 +9,8 @@
|
||||
#include <cstring>
|
||||
#include <utility>
|
||||
|
||||
#include "io/platform/file_ops.hpp"
|
||||
|
||||
namespace vdm::io {
|
||||
namespace {
|
||||
|
||||
@@ -90,20 +92,12 @@ Result<void> SparseFile::open(std::string_view path, const OpenOptions &opts) {
|
||||
bool prealloc = false;
|
||||
if (opts.total_size > 0) {
|
||||
if (opts.preallocate) {
|
||||
// posix_fallocate returns the error number directly and does not set errno.
|
||||
int rc = ::posix_fallocate(fd, 0, static_cast<off_t>(opts.total_size));
|
||||
if (rc == 0) {
|
||||
prealloc = true;
|
||||
} else if (rc == EOPNOTSUPP || rc == ENOSYS || rc == EINVAL) {
|
||||
if (::ftruncate(fd, static_cast<off_t>(opts.total_size)) != 0) {
|
||||
int e = errno;
|
||||
Result<bool> r = platform::preallocate(fd, opts.total_size);
|
||||
if (!r.has_value()) {
|
||||
::close(fd);
|
||||
return sys_error("ftruncate " + p, e);
|
||||
}
|
||||
} else {
|
||||
::close(fd);
|
||||
return sys_error("posix_fallocate " + p, rc);
|
||||
return std::move(r).error();
|
||||
}
|
||||
prealloc = *r;
|
||||
} else if (!opts.truncate_existing) {
|
||||
// Resuming: make sure the file is at least total_size so pwrite offsets land.
|
||||
if (::ftruncate(fd, static_cast<off_t>(opts.total_size)) != 0) {
|
||||
@@ -147,18 +141,13 @@ Result<void> SparseFile::write_at(std::uint64_t offset, ConstByteSpan data) {
|
||||
Result<void> SparseFile::sync() {
|
||||
if (fd_ < 0)
|
||||
return ErrorInfo(Error::internal, "sync on a closed SparseFile");
|
||||
while (::fdatasync(fd_) != 0) {
|
||||
if (errno == EINTR)
|
||||
continue;
|
||||
return sys_error("fdatasync", errno);
|
||||
}
|
||||
return ok();
|
||||
return platform::flush_durable(fd_);
|
||||
}
|
||||
|
||||
void SparseFile::advise_dontneed(std::uint64_t offset, std::uint64_t len) noexcept {
|
||||
if (fd_ < 0 || len == 0)
|
||||
return;
|
||||
::posix_fadvise(fd_, static_cast<off_t>(offset), static_cast<off_t>(len), POSIX_FADV_DONTNEED);
|
||||
platform::advise_dontneed(fd_, offset, len);
|
||||
}
|
||||
|
||||
Result<void> SparseFile::resize(std::uint64_t size) {
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include "io/platform/file_ops.hpp"
|
||||
#include "vdm/util/crc32.hpp"
|
||||
|
||||
namespace vdm::meta {
|
||||
@@ -272,13 +273,11 @@ Result<void> write_veloxpart_file(std::string_view path, const VeloxPart &vp, bo
|
||||
}
|
||||
|
||||
if (fsync) {
|
||||
while (::fdatasync(fd) != 0) {
|
||||
if (errno == EINTR)
|
||||
continue;
|
||||
int e = errno;
|
||||
Result<void> r = vdm::io::platform::flush_durable(fd);
|
||||
if (!r.has_value()) {
|
||||
::close(fd);
|
||||
::unlink(tmp.c_str());
|
||||
return sys_error("fdatasync " + tmp, e);
|
||||
return ErrorInfo(r.error().code, "fdatasync " + tmp + ": " + r.error().context);
|
||||
}
|
||||
}
|
||||
if (::close(fd) != 0) {
|
||||
|
||||
@@ -290,11 +290,20 @@ struct HttpClient::Impl {
|
||||
}
|
||||
|
||||
void drain_commands(Worker &w) {
|
||||
// Called every worker-loop iteration (run(), below) -- once per curl_multi_poll
|
||||
// wake, so once per socket-readiness event on the transfer hot path -- but commands
|
||||
// (add/pause/resume/cancel) are rare next to that. Check empty under the lock
|
||||
// *before* touching `local`: libstdc++'s std::deque allocates its map array on
|
||||
// default construction even with nothing pushed to it, so constructing one every
|
||||
// iteration just to usually swap nothing into it was an allocation on every poll
|
||||
// wake, not just on an actual command -- exactly what the curl-write-callback path
|
||||
// must never do (AGENT-CORE.md; caught by tools/bench's alloc-check).
|
||||
std::unique_lock lk(w.mu);
|
||||
if (w.queue.empty())
|
||||
return;
|
||||
std::deque<Command> local;
|
||||
{
|
||||
std::lock_guard lk(w.mu);
|
||||
local.swap(w.queue);
|
||||
}
|
||||
lk.unlock();
|
||||
for (auto &cmd : local) {
|
||||
auto &st = cmd.state;
|
||||
switch (cmd.kind) {
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
#include "vdm/rules/collision.hpp"
|
||||
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace vdm::rules {
|
||||
namespace {
|
||||
|
||||
// Split on the last '.', unless it's a leading dot (a dotfile like ".gitignore" has no
|
||||
// extension by this convention — matches sanitize_filename's own treatment). Returns
|
||||
// {stem, ext} where `ext` includes the leading '.' when present.
|
||||
std::pair<std::string_view, std::string_view> split_stem_ext(std::string_view name) {
|
||||
auto dot = name.rfind('.');
|
||||
if (dot == std::string_view::npos || dot == 0)
|
||||
return {name, {}};
|
||||
return {name.substr(0, dot), name.substr(dot)};
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string resolve_collision(std::string_view desired,
|
||||
const std::function<bool(std::string_view)> &exists,
|
||||
CollisionPolicy policy, int max_attempts) {
|
||||
if (policy == CollisionPolicy::overwrite)
|
||||
return std::string(desired);
|
||||
|
||||
if (!exists || !exists(desired))
|
||||
return std::string(desired);
|
||||
|
||||
auto [stem, ext] = split_stem_ext(desired);
|
||||
std::string candidate;
|
||||
for (int n = 1; n <= max_attempts; ++n) {
|
||||
candidate = std::string(stem) + " (" + std::to_string(n) + ")" + std::string(ext);
|
||||
if (!exists(candidate))
|
||||
return candidate;
|
||||
}
|
||||
return candidate; // still colliding; see the header comment on why this isn't hidden
|
||||
}
|
||||
|
||||
} // namespace vdm::rules
|
||||
@@ -0,0 +1,127 @@
|
||||
#include "vdm/rules/filename.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cctype>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace vdm::rules {
|
||||
namespace {
|
||||
|
||||
bool is_control_byte(unsigned char c) { return c < 0x20 || c == 0x7F; }
|
||||
|
||||
bool is_ntfs_illegal(char c) {
|
||||
switch (c) {
|
||||
case '<':
|
||||
case '>':
|
||||
case ':':
|
||||
case '"':
|
||||
case '|':
|
||||
case '?':
|
||||
case '*':
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// CON, PRN, AUX, NUL, COM1-9, LPT1-9 — Win32 device names, matched case-insensitively.
|
||||
// `stem` is already ASCII-only by the time this runs (everything else has been filtered),
|
||||
// so a byte-wise toupper is enough; no locale, no UTF-8 concerns.
|
||||
bool is_reserved_device_name(std::string_view stem) {
|
||||
static constexpr std::array<std::string_view, 4> kFixed = {"CON", "PRN", "AUX", "NUL"};
|
||||
std::string upper(stem);
|
||||
std::transform(upper.begin(), upper.end(), upper.begin(),
|
||||
[](unsigned char c) { return static_cast<char>(std::toupper(c)); });
|
||||
for (auto f : kFixed)
|
||||
if (upper == f)
|
||||
return true;
|
||||
if (upper.size() == 4 && (upper.starts_with("COM") || upper.starts_with("LPT")) &&
|
||||
upper[3] >= '1' && upper[3] <= '9')
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
// Backs `pos` up off any UTF-8 continuation bytes (10xxxxxx) so a byte-length cut never
|
||||
// splits a multi-byte codepoint. `pos` is a candidate cut index into `s`, 0 <= pos <=
|
||||
// s.size().
|
||||
std::size_t utf8_safe_cut(std::string_view s, std::size_t pos) {
|
||||
while (pos > 0 && (static_cast<unsigned char>(s[pos]) & 0xC0) == 0x80)
|
||||
--pos;
|
||||
return pos;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string sanitize_filename(std::string_view raw, std::size_t max_bytes) {
|
||||
// Pass 1: drop control bytes and path separators outright; fold the other
|
||||
// NTFS-illegal characters to '_'. Everything else (including non-ASCII UTF-8) passes
|
||||
// through untouched.
|
||||
std::string s;
|
||||
s.reserve(raw.size());
|
||||
for (char c : raw) {
|
||||
auto uc = static_cast<unsigned char>(c);
|
||||
if (is_control_byte(uc) || c == '/' || c == '\\')
|
||||
continue;
|
||||
s.push_back(is_ntfs_illegal(c) ? '_' : c);
|
||||
}
|
||||
|
||||
// Pass 2: collapse any run of 2+ '.' down to one, so stripped separators can't
|
||||
// reconstitute a ".." (or longer) traversal-looking sequence out of what's left.
|
||||
{
|
||||
std::string collapsed;
|
||||
collapsed.reserve(s.size());
|
||||
for (std::size_t i = 0; i < s.size(); ++i) {
|
||||
if (s[i] == '.' && i > 0 && collapsed.size() > 0 && collapsed.back() == '.')
|
||||
continue;
|
||||
collapsed.push_back(s[i]);
|
||||
}
|
||||
s = std::move(collapsed);
|
||||
}
|
||||
|
||||
// Pass 3: strip trailing '.' and ' ' — both are silently dropped by the Win32 API, so
|
||||
// leaving them lets two different requested names collide invisibly on export.
|
||||
while (!s.empty() && (s.back() == '.' || s.back() == ' '))
|
||||
s.pop_back();
|
||||
|
||||
if (s.empty())
|
||||
s = "download";
|
||||
|
||||
// Pass 4: reserved device name check, against the part before the first '.' (or the
|
||||
// whole name if there's none). The '_' goes right after the stem, before any
|
||||
// extension ("NUL.txt" -> "NUL_.txt"), so the result still looks like the same kind of
|
||||
// file rather than growing a spurious trailing character after its extension.
|
||||
{
|
||||
auto dot = s.find('.');
|
||||
std::string_view stem = dot != std::string::npos ? std::string_view(s).substr(0, dot)
|
||||
: std::string_view(s);
|
||||
if (is_reserved_device_name(stem))
|
||||
s.insert(stem.size(), "_");
|
||||
}
|
||||
|
||||
// Pass 5: clamp to max_bytes, UTF-8-safe, keeping a short trailing extension intact
|
||||
// where possible.
|
||||
if (s.size() > max_bytes) {
|
||||
std::string_view ext;
|
||||
if (auto dot = s.rfind('.'); dot != std::string::npos && dot > 0 && s.size() - dot <= 16)
|
||||
ext = std::string_view(s).substr(dot);
|
||||
if (ext.size() < max_bytes) {
|
||||
std::size_t stem_budget = max_bytes - ext.size();
|
||||
std::size_t cut = utf8_safe_cut(s, stem_budget);
|
||||
s = s.substr(0, cut) + std::string(ext);
|
||||
} else {
|
||||
s = s.substr(0, utf8_safe_cut(s, max_bytes));
|
||||
}
|
||||
// Re-strip: truncation can expose a new trailing '.'/' ' (e.g. the byte right
|
||||
// before the cut was itself a dot that pass 3 had no reason to touch).
|
||||
while (!s.empty() && (s.back() == '.' || s.back() == ' '))
|
||||
s.pop_back();
|
||||
if (s.empty())
|
||||
s = "download";
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace vdm::rules
|
||||
@@ -0,0 +1,98 @@
|
||||
#include "vdm/rules/match.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cstddef>
|
||||
#include <numeric>
|
||||
|
||||
namespace vdm::rules {
|
||||
namespace {
|
||||
|
||||
char lower_ascii(char c) { return static_cast<char>(std::tolower(static_cast<unsigned char>(c))); }
|
||||
|
||||
bool ieq(std::string_view a, std::string_view b) {
|
||||
return a.size() == b.size() &&
|
||||
std::equal(a.begin(), a.end(), b.begin(),
|
||||
[](char x, char y) { return lower_ascii(x) == lower_ascii(y); });
|
||||
}
|
||||
|
||||
// A rule's `extensions` entries are documented with no leading '.', but be lenient about
|
||||
// one showing up anyway (a hand-edited rule table, an older client) rather than let a
|
||||
// clause that never matches silently swallow a whole category.
|
||||
std::string_view strip_leading_dot(std::string_view s) {
|
||||
return (!s.empty() && s.front() == '.') ? s.substr(1) : s;
|
||||
}
|
||||
|
||||
bool match_clause(const RuleMatch &m, const MatchInput &in) {
|
||||
if (m.extensions) {
|
||||
bool any = std::any_of(m.extensions->begin(), m.extensions->end(), [&](const auto &e) {
|
||||
return ieq(strip_leading_dot(e), in.extension);
|
||||
});
|
||||
if (!any)
|
||||
return false;
|
||||
}
|
||||
if (m.mime_types) {
|
||||
bool any = std::any_of(m.mime_types->begin(), m.mime_types->end(),
|
||||
[&](const auto &t) { return ieq(t, in.mime_type); });
|
||||
if (!any)
|
||||
return false;
|
||||
}
|
||||
if (m.host_pattern && !glob_match(*m.host_pattern, in.host))
|
||||
return false;
|
||||
if (m.url_pattern && !glob_match(*m.url_pattern, in.url))
|
||||
return false;
|
||||
if (m.min_size_bytes) {
|
||||
if (!in.size_bytes || *in.size_bytes < *m.min_size_bytes)
|
||||
return false;
|
||||
}
|
||||
if (m.max_size_bytes) {
|
||||
if (!in.size_bytes || *in.size_bytes > *m.max_size_bytes)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool glob_match(std::string_view pattern, std::string_view text) noexcept {
|
||||
// Classic iterative wildcard match (single backtrack point at the most recent '*'), not
|
||||
// the naive recursive version — bounded work on any input, including a pattern that is
|
||||
// nothing but repeated '*'s against a long `text`.
|
||||
std::size_t p = 0, t = 0;
|
||||
std::size_t star = std::string_view::npos, mark = 0;
|
||||
while (t < text.size()) {
|
||||
if (p < pattern.size() && (pattern[p] == '?' || lower_ascii(pattern[p]) == lower_ascii(text[t]))) {
|
||||
++p;
|
||||
++t;
|
||||
} else if (p < pattern.size() && pattern[p] == '*') {
|
||||
star = p++;
|
||||
mark = t;
|
||||
} else if (star != std::string_view::npos) {
|
||||
p = star + 1;
|
||||
t = ++mark;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
while (p < pattern.size() && pattern[p] == '*') ++p;
|
||||
return p == pattern.size();
|
||||
}
|
||||
|
||||
std::optional<RuleAction> match_rules(const std::vector<Rule> &rules, const MatchInput &input) {
|
||||
std::vector<std::size_t> order(rules.size());
|
||||
std::iota(order.begin(), order.end(), 0);
|
||||
// Stable by construction: std::stable_sort keeps table order among equal priorities.
|
||||
std::stable_sort(order.begin(), order.end(), [&](std::size_t a, std::size_t b) {
|
||||
return rules[a].priority < rules[b].priority;
|
||||
});
|
||||
for (auto i : order) {
|
||||
const Rule &r = rules[i];
|
||||
if (!r.enabled)
|
||||
continue;
|
||||
if (match_clause(r.match, input))
|
||||
return r.action;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
} // namespace vdm::rules
|
||||
@@ -37,12 +37,10 @@ SegmentBudget::EngineBudget SegmentBudget::snapshot_locked() const {
|
||||
return EngineBudget{max_active_, active_, starved};
|
||||
}
|
||||
|
||||
// The two-pass fairness allocation. Recomputes every task's target from scratch (so a
|
||||
// live cap cut naturally produces target < held -> yield), diffs against the last
|
||||
// published target, and collects the callbacks to fire once mu_ is released.
|
||||
SegmentBudget::Plan SegmentBudget::reallocate_locked() {
|
||||
// Priority order: DAEMON's list first, then any registered task not in it (defensive;
|
||||
// "a running task absent from the list sorts last").
|
||||
// DAEMON's list first, then any registered task not in it (defensive; "a running task
|
||||
// absent from the list sorts last"). Shared by reallocate_locked() and
|
||||
// wake_one_waiter_locked(), which need the same priority ordering.
|
||||
std::vector<TaskId> SegmentBudget::priority_order_locked() const {
|
||||
std::vector<TaskId> order;
|
||||
order.reserve(tasks_.size());
|
||||
for (TaskId id : order_)
|
||||
@@ -51,6 +49,14 @@ SegmentBudget::Plan SegmentBudget::reallocate_locked() {
|
||||
for (const auto &[id, _] : tasks_)
|
||||
if (std::find(order.begin(), order.end(), id) == order.end())
|
||||
order.push_back(id);
|
||||
return order;
|
||||
}
|
||||
|
||||
// The two-pass fairness allocation. Recomputes every task's target from scratch (so a
|
||||
// live cap cut naturally produces target < held -> yield), diffs against the last
|
||||
// published target, and collects the callbacks to fire once mu_ is released.
|
||||
SegmentBudget::Plan SegmentBudget::reallocate_locked() {
|
||||
std::vector<TaskId> order = priority_order_locked();
|
||||
|
||||
std::unordered_map<TaskId, std::uint32_t> target;
|
||||
target.reserve(order.size());
|
||||
@@ -155,6 +161,7 @@ void SegmentBudget::deregister_task(TaskId id) {
|
||||
active_ -= it->second.held;
|
||||
tasks_.erase(it);
|
||||
plan = reallocate_locked();
|
||||
wake_one_waiter_locked(plan, id); // a departing task frees real slots too
|
||||
}
|
||||
run(plan);
|
||||
}
|
||||
@@ -182,6 +189,19 @@ bool SegmentBudget::confirm_slot(TaskId id) {
|
||||
Task &t = it->second;
|
||||
if (t.held >= t.target)
|
||||
return false; // target was cut in the race
|
||||
// reallocate_locked()'s pool math bounds sum(target) <= max_active_ *as computed*, but
|
||||
// that doesn't bound sum(held): a task can be legitimately over its own just-lowered
|
||||
// target for a while (yield deferred to a segment boundary, ADR 0011 A1), and another
|
||||
// task's target can correctly rise to claim that capacity before the first task has
|
||||
// physically released it. active_ is the one number that's always true regardless of
|
||||
// any task's target bookkeeping, so it's the backstop. Denials here are remembered
|
||||
// (waiting_for_slot) rather than left for the caller to somehow ask again at the right
|
||||
// moment -- see release_slot()'s wake_one_waiter_locked() call.
|
||||
if (active_ >= max_active_) {
|
||||
t.waiting_for_slot = true;
|
||||
return false;
|
||||
}
|
||||
t.waiting_for_slot = false;
|
||||
++t.held;
|
||||
++active_;
|
||||
if (snapshot_locked() != last_notified_) {
|
||||
@@ -191,6 +211,25 @@ bool SegmentBudget::confirm_slot(TaskId id) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Appends a retry hint for the highest-priority task with waiting_for_slot set (other
|
||||
// than `exclude`) to `plan`. Only ever wakes a task confirm_slot() actually turned away --
|
||||
// not just anyone below its target, which would also fire for tasks that are fairly,
|
||||
// correctly not entitled to more right now (see reallocate_locked()'s own target math).
|
||||
void SegmentBudget::wake_one_waiter_locked(Plan &plan, TaskId exclude) const {
|
||||
for (TaskId id : priority_order_locked()) {
|
||||
if (id == exclude)
|
||||
continue;
|
||||
auto it = tasks_.find(id);
|
||||
if (it == tasks_.end())
|
||||
continue;
|
||||
const Task &t = it->second;
|
||||
if (t.waiting_for_slot && t.on_target) {
|
||||
plan.targets.emplace_back(t.on_target, t.target);
|
||||
return; // exactly one freed slot, exactly one retry hint
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void SegmentBudget::release_slot(TaskId id) {
|
||||
Plan plan;
|
||||
{
|
||||
@@ -201,6 +240,11 @@ void SegmentBudget::release_slot(TaskId id) {
|
||||
--it->second.held;
|
||||
--active_;
|
||||
plan = reallocate_locked();
|
||||
// reallocate_locked() only fires a callback for a task whose *target* changed.
|
||||
// The task this freed slot is actually owed to (see confirm_slot()) may have a
|
||||
// target that was already correct and hasn't moved -- nothing else will ever ask
|
||||
// it to retry, so the budget has to remember and hand this slot to it directly.
|
||||
wake_one_waiter_locked(plan, id);
|
||||
}
|
||||
run(plan);
|
||||
}
|
||||
|
||||
+374
-75
@@ -15,6 +15,7 @@
|
||||
#include <atomic>
|
||||
#include <cctype>
|
||||
#include <cerrno>
|
||||
#include <condition_variable>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
@@ -63,8 +64,32 @@ std::string lower(std::string s) {
|
||||
return s;
|
||||
}
|
||||
|
||||
// scheme://host[:port] of `url`, with no path/query/fragment -- what docs/04 §7's "403
|
||||
// after redirect: retry once with the original referrer" retries with as the Referer
|
||||
// header. Empty on an unparseable URL (the caller just won't get a referrer retry).
|
||||
std::string origin_of(std::string_view url) {
|
||||
auto s = net::split_url(url);
|
||||
if (!s.valid)
|
||||
return {};
|
||||
std::string out = s.scheme + "://" + s.host;
|
||||
if (s.port)
|
||||
out += ":" + std::to_string(*s.port);
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// What to do once every worker has drained (see DownloadTaskState::begin_drain_locked).
|
||||
// A worker that hits one of these outcomes must not tear the task down itself: siblings may
|
||||
// still be mid-transfer, and cutting them off synchronously — cancel every worker, clear the
|
||||
// map, proceed — drops their buffered-but-unflushed bytes while segment_completed() already
|
||||
// counts those bytes as done. That's exactly the class of bug that makes a later resume skip
|
||||
// real data (seg_data() runs append() without workers_mu, so a sibling's buffer can't be
|
||||
// flushed safely from here anyway). Instead: cancel the siblings, remember what to do, and
|
||||
// let each one's own seg_finished (which already flushes on every exit path) run it once the
|
||||
// worker map is actually empty.
|
||||
enum class PendingAction { none, verify, fail, auto_pause, demote };
|
||||
|
||||
struct SegWorker {
|
||||
std::uint32_t seg_index = 0;
|
||||
net::Transfer transfer;
|
||||
@@ -76,14 +101,26 @@ struct SegWorker {
|
||||
bool needs_auth = false;
|
||||
bool wrong_status = false;
|
||||
bool range_bad = false;
|
||||
bool forbidden = false; // 403 -- docs/04 §7's "retry once with the original referrer"
|
||||
bool auth_handshake = false; // saw a 401/407 and let libcurl resend with credentials
|
||||
std::string resp_etag, resp_last_modified; // captured on a wrong_status 200, for demote
|
||||
std::optional<ErrorInfo> flush_error;
|
||||
|
||||
int retries = 0;
|
||||
|
||||
SteadyTime sample_at{};
|
||||
std::uint64_t sample_bytes = 0;
|
||||
double speed_bps = 0;
|
||||
// Written only by this segment's own curl callback (seg_data, sequential -- no lock
|
||||
// held across the update, by design: the transfer hot path takes no lock it doesn't
|
||||
// need). snapshot_progress() reads it from whatever thread calls
|
||||
// DownloadHandle::progress() (DAEMON's polling, or anyone else's), which workers_mu's
|
||||
// shared_lock does NOT cover -- that lock only protects the `workers` map's own
|
||||
// structure, not an individual SegWorker's mutable fields. atomic<double> (relaxed:
|
||||
// this is an approximate, informational EMA, not something anything synchronizes
|
||||
// real state on) keeps that read-from-any-thread safe without adding a lock to the
|
||||
// write side. Found by TSan the first time anything actually read this cross-thread
|
||||
// (engine_polled_progress_reports_nonzero_speed, added alongside the speed_bps fix).
|
||||
std::atomic<double> speed_bps{0};
|
||||
};
|
||||
|
||||
struct DownloadTaskState : std::enable_shared_from_this<DownloadTaskState> {
|
||||
@@ -95,6 +132,9 @@ struct DownloadTaskState : std::enable_shared_from_this<DownloadTaskState> {
|
||||
std::mutex mu;
|
||||
std::shared_mutex workers_mu;
|
||||
std::mutex deferred_mu;
|
||||
// Notified (holding mu) whenever seg_finished removes an entry from `workers`; quiesce()
|
||||
// waits on it instead of clearing the map itself — see quiesce()'s comment.
|
||||
std::condition_variable workers_drained_cv;
|
||||
|
||||
EngineState state = EngineState::probing;
|
||||
std::optional<ErrorInfo> last_error;
|
||||
@@ -109,6 +149,16 @@ struct DownloadTaskState : std::enable_shared_from_this<DownloadTaskState> {
|
||||
std::optional<std::uint64_t> total_size;
|
||||
std::string origin_host;
|
||||
|
||||
// docs/04 §7's "403 after redirect: retry once with the original referrer" -- many
|
||||
// CDNs 403 a bare/foreign Referer. Starts as spec.referrer (the browser's, verbatim);
|
||||
// start_worker_locked() sends this, not spec.referrer directly, so a 403 retry can
|
||||
// override it (to the download URL's own origin) without touching what the caller
|
||||
// actually asked for. referrer_retried bounds it to exactly once per task -- a second
|
||||
// 403 with a same-origin Referer already set is a real, honest failure
|
||||
// (Error::forbidden), not something a referrer swap can fix.
|
||||
std::string effective_referrer;
|
||||
bool referrer_retried = false;
|
||||
|
||||
std::unique_ptr<segment::Segmenter> seg;
|
||||
std::unique_ptr<io::SparseFile> file;
|
||||
std::unordered_map<std::uint32_t, std::unique_ptr<SegWorker>> workers;
|
||||
@@ -120,19 +170,25 @@ struct DownloadTaskState : std::enable_shared_from_this<DownloadTaskState> {
|
||||
|
||||
bool pause_requested = false;
|
||||
bool cancel_requested = false;
|
||||
bool assembling = false; // every byte received; draining live workers' buffers to disk
|
||||
bool discard_on_cancel = false;
|
||||
bool awaiting_auth = false;
|
||||
bool awaiting_decision = false;
|
||||
int max_retries = 10;
|
||||
|
||||
// Set by begin_drain_locked while waiting for sibling workers to drain; see
|
||||
// PendingAction above.
|
||||
PendingAction pending_action = PendingAction::none;
|
||||
std::optional<ErrorInfo> pending_error;
|
||||
bool pending_auth = false;
|
||||
bool pending_decision = false;
|
||||
|
||||
std::atomic<std::int64_t> last_progress_ns{0};
|
||||
SteadyTime started_at{};
|
||||
|
||||
std::vector<std::function<void()>> deferred;
|
||||
|
||||
DownloadTaskState(TaskHost &h, TaskId i, DownloadSpec s, DownloadCallbacks c)
|
||||
: host(h), id(i), spec(std::move(s)), cbs(std::move(c)) {}
|
||||
: host(h), id(i), spec(std::move(s)), cbs(std::move(c)), effective_referrer(spec.referrer) {}
|
||||
|
||||
// --- deferred callbacks -------------------------------------------------------------
|
||||
void defer(std::function<void()> fn) {
|
||||
@@ -188,6 +244,12 @@ struct DownloadTaskState : std::enable_shared_from_this<DownloadTaskState> {
|
||||
void on_probe_result(Result<net::ProbeResult> r);
|
||||
void finish_probe_locked();
|
||||
void apply_slot_target(std::uint32_t n);
|
||||
void fill_slots_locked();
|
||||
// Confirms a budget slot and starts a worker for `seg_idx` if nothing already covers
|
||||
// it. false on either failure (already running, or budget denied) -- the caller's own
|
||||
// fallback (fill_slots_locked's assign_slot() loop, retry_worker()'s early return)
|
||||
// stays the same either way. Caller holds mu.
|
||||
bool try_start_segment_locked(std::uint32_t seg_idx);
|
||||
void start_worker_locked(std::uint32_t seg_idx);
|
||||
void restart_probe(bool with_auth);
|
||||
|
||||
@@ -200,8 +262,8 @@ struct DownloadTaskState : std::enable_shared_from_this<DownloadTaskState> {
|
||||
void begin_verify_locked();
|
||||
void fail_locked(ErrorInfo e);
|
||||
void auto_pause_locked(ErrorInfo e, bool auth, bool decision);
|
||||
void cancel_all_transfers_locked();
|
||||
void start_assembly_locked();
|
||||
void demote_to_single_segment_locked();
|
||||
void begin_drain_locked(PendingAction action, ErrorInfo e, bool auth, bool decision);
|
||||
void finalize_cancel_locked();
|
||||
void write_sidecar_locked();
|
||||
void emit_progress_if_due();
|
||||
@@ -248,7 +310,7 @@ void DownloadTaskState::restart_probe(bool with_auth) {
|
||||
pr.url = spec.url;
|
||||
pr.headers = spec.headers;
|
||||
pr.cookies = spec.cookies;
|
||||
pr.referrer = spec.referrer;
|
||||
pr.referrer = effective_referrer;
|
||||
pr.user_agent = spec.user_agent;
|
||||
pr.proxy = spec.proxy;
|
||||
if (with_auth)
|
||||
@@ -261,12 +323,37 @@ void DownloadTaskState::restart_probe(bool with_auth) {
|
||||
}
|
||||
|
||||
void DownloadTaskState::on_probe_result(Result<net::ProbeResult> r) {
|
||||
bool retry_probe_with_referrer = false;
|
||||
{
|
||||
std::unique_lock lk(mu);
|
||||
if (retired.load() || is_terminal(state))
|
||||
return;
|
||||
if (!r.has_value()) {
|
||||
fail_locked(std::move(r).error());
|
||||
ErrorInfo e = std::move(r).error();
|
||||
// docs/04 §7's referrer retry applies here too: a probe (HEAD, or the
|
||||
// ranged-GET fallback when HEAD is refused -- probe.cpp) can be the request
|
||||
// that actually gets 403'd, before any segment worker exists to retry it
|
||||
// (net::Prober builds its own request from ProbeRequest::referrer, not
|
||||
// through start_worker_locked() -- see restart_probe()'s use of
|
||||
// effective_referrer below). Same one-shot bound via referrer_retried as the
|
||||
// worker-level retry (seg_finished's w->forbidden branch) shares.
|
||||
if (e.code == Error::forbidden && !referrer_retried) {
|
||||
referrer_retried = true;
|
||||
effective_referrer = origin_of(spec.url);
|
||||
retry_probe_with_referrer = true;
|
||||
} else if (e.code == Error::forbidden) {
|
||||
// Already retried with the origin referrer and still 403 -- not something
|
||||
// another blind retry fixes (an expired signed URL, a private resource).
|
||||
// Ask rather than fail outright, the same "ask, don't just fail" shape as
|
||||
// wrong_status/range_bad/the worker-level 403 branch: refresh_url() is a
|
||||
// no-op once the task is terminal, and tools/testserver's expiring-signed-
|
||||
// url mode (also a bare 403, indistinguishable from any other without
|
||||
// parsing the body -- CLAUDE.md §3, core never does) is meant to be
|
||||
// recovered exactly that way.
|
||||
auto_pause_locked(std::move(e), false, true);
|
||||
} else {
|
||||
fail_locked(std::move(e));
|
||||
}
|
||||
} else {
|
||||
probe = std::move(r).value();
|
||||
have_probe = true;
|
||||
@@ -281,6 +368,8 @@ void DownloadTaskState::on_probe_result(Result<net::ProbeResult> r) {
|
||||
}
|
||||
}
|
||||
flush_deferred();
|
||||
if (retry_probe_with_referrer)
|
||||
restart_probe(false);
|
||||
}
|
||||
|
||||
void DownloadTaskState::finish_probe_locked() {
|
||||
@@ -345,13 +434,39 @@ void DownloadTaskState::finish_probe_locked() {
|
||||
host.budget().set_want(id, want_slots());
|
||||
}
|
||||
|
||||
void DownloadTaskState::apply_slot_target(std::uint32_t n) {
|
||||
{
|
||||
std::unique_lock lk(mu);
|
||||
if (retired.load() || is_terminal(state) || pause_requested || cancel_requested ||
|
||||
awaiting_auth || awaiting_decision || assembling || !seg)
|
||||
return;
|
||||
slot_target = n;
|
||||
bool DownloadTaskState::try_start_segment_locked(std::uint32_t seg_idx) {
|
||||
if (workers.count(seg_idx))
|
||||
return false;
|
||||
if (!host.budget().confirm_slot(id))
|
||||
return false;
|
||||
start_worker_locked(seg_idx);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Start workers up to `slot_target`, given whatever the budget currently confirms. Shared
|
||||
// by apply_slot_target() (the budget's async callback, whenever the computed target
|
||||
// actually changes) and demote_to_single_segment_locked() -- the demoted target can
|
||||
// legitimately equal what it was before the rebuild (e.g. a download already down to its
|
||||
// last segment), in which case SegmentBudget::set_want() no-ops and the async callback
|
||||
// never fires, so nothing else would ever start the replacement worker.
|
||||
void DownloadTaskState::fill_slots_locked() {
|
||||
// A segment mid-backoff (SegState::stalled -- seg_finished's retry path: released its
|
||||
// slot, scheduled a retry_worker() timer, and gave up quietly if confirm_slot() denied
|
||||
// it then) has no live worker and never surfaces through assign_slot() -- it stays
|
||||
// assigned to whichever segment iteration created it, just not running. It's not
|
||||
// "fresh work" the loop below would ever find on its own. Every path that can free
|
||||
// budget capacity ends up here (apply_slot_target(), driven by SegmentBudget's
|
||||
// target-changed callback *and* its wait-list wakeup for a task whose target didn't
|
||||
// move -- see confirm_slot()/release_slot() in budget.cpp), so this is the one place
|
||||
// that needs to give a stalled segment another try, not every caller of
|
||||
// retry_worker(). Bounded by slot_target like the assign_slot() loop below; a segment
|
||||
// this doesn't get to keeps its own scheduled retry_worker() timer as a second chance.
|
||||
for (auto &v : seg->snapshot()) {
|
||||
if (workers.size() >= slot_target)
|
||||
break;
|
||||
if (v.state == segment::SegState::stalled)
|
||||
try_start_segment_locked(v.index);
|
||||
}
|
||||
while (workers.size() < slot_target) {
|
||||
auto s = seg->assign_slot();
|
||||
if (!s) {
|
||||
@@ -364,9 +479,23 @@ void DownloadTaskState::apply_slot_target(std::uint32_t n) {
|
||||
}
|
||||
start_worker_locked(*s);
|
||||
}
|
||||
if (state == EngineState::connecting && !workers.empty())
|
||||
// Mirrors retry_worker()'s own transition: a stalled-segment restart above can be the
|
||||
// thing that takes a retry_wait task back to actually transferring, same as connecting
|
||||
// does for a task starting up.
|
||||
if ((state == EngineState::connecting || state == EngineState::retry_wait) &&
|
||||
!workers.empty())
|
||||
transition(EngineState::downloading, std::nullopt);
|
||||
}
|
||||
|
||||
void DownloadTaskState::apply_slot_target(std::uint32_t n) {
|
||||
{
|
||||
std::unique_lock lk(mu);
|
||||
if (retired.load() || is_terminal(state) || pause_requested || cancel_requested ||
|
||||
awaiting_auth || awaiting_decision || pending_action != PendingAction::none || !seg)
|
||||
return;
|
||||
slot_target = n;
|
||||
fill_slots_locked();
|
||||
}
|
||||
flush_deferred();
|
||||
}
|
||||
|
||||
@@ -394,7 +523,7 @@ void DownloadTaskState::start_worker_locked(std::uint32_t seg_idx) {
|
||||
req.url = current_url();
|
||||
req.headers = spec.headers;
|
||||
req.cookies = spec.cookies;
|
||||
req.referrer = spec.referrer;
|
||||
req.referrer = effective_referrer;
|
||||
req.user_agent = spec.user_agent;
|
||||
req.proxy = spec.proxy;
|
||||
req.auth = spec.auth;
|
||||
@@ -464,12 +593,20 @@ net::DataAction DownloadTaskState::seg_head(std::uint32_t seg_idx, const net::Re
|
||||
// gets a 200 (the source has no Range support).
|
||||
if (resumable && total_size && *total_size > 0 && h.status == 200) {
|
||||
w->wrong_status = true;
|
||||
if (auto v = h.headers.get("ETag"))
|
||||
w->resp_etag.assign(*v);
|
||||
if (auto v = h.headers.get("Last-Modified"))
|
||||
w->resp_last_modified.assign(*v);
|
||||
return net::DataAction::abort;
|
||||
}
|
||||
if (h.status == 416) {
|
||||
w->range_bad = true;
|
||||
return net::DataAction::abort;
|
||||
}
|
||||
if (h.status == 403) {
|
||||
w->forbidden = true;
|
||||
return net::DataAction::abort;
|
||||
}
|
||||
if (h.status >= 400)
|
||||
return net::DataAction::abort;
|
||||
seg->set_segment_state(seg_idx, segment::SegState::downloading);
|
||||
@@ -510,7 +647,9 @@ net::DataAction DownloadTaskState::seg_data(std::uint32_t seg_idx, ConstByteSpan
|
||||
auto dt = std::chrono::duration<double>(now - w->sample_at).count();
|
||||
if (dt >= 0.5) {
|
||||
double inst = static_cast<double>(w->recv - w->sample_bytes) / dt;
|
||||
w->speed_bps = w->speed_bps == 0 ? inst : 0.7 * w->speed_bps + 0.3 * inst;
|
||||
double prev = w->speed_bps.load(std::memory_order_relaxed);
|
||||
w->speed_bps.store(prev == 0 ? inst : 0.7 * prev + 0.3 * inst,
|
||||
std::memory_order_relaxed);
|
||||
w->sample_at = now;
|
||||
w->sample_bytes = w->recv;
|
||||
}
|
||||
@@ -541,6 +680,7 @@ void DownloadTaskState::seg_finished(std::uint32_t seg_idx, Result<net::Transfer
|
||||
w = std::move(it->second);
|
||||
workers.erase(it);
|
||||
}
|
||||
workers_drained_cv.notify_all(); // quiesce() may be waiting for `workers` to empty out
|
||||
if (retired.load()) { // engine shutting down / already terminal — no more callbacks
|
||||
if (w->buf)
|
||||
(void)w->buf->flush();
|
||||
@@ -552,7 +692,7 @@ void DownloadTaskState::seg_finished(std::uint32_t seg_idx, Result<net::Transfer
|
||||
// (content-length-mismatch's honest-length lie, flaky-reset's tail, a proxy RST after
|
||||
// the last byte). If the segment is fully covered, that's a success.
|
||||
if (seg && !cancel_requested && !pause_requested && !w->needs_auth && !w->wrong_status &&
|
||||
!w->flush_error && !w->range_bad) {
|
||||
!w->flush_error && !w->range_bad && !w->forbidden) {
|
||||
const std::uint64_t len = seg->segment_end(seg_idx) - seg->segment_start(seg_idx) + 1;
|
||||
if (len != 0 && seg->segment_completed(seg_idx) >= len) {
|
||||
r = Result<net::TransferStats>(net::TransferStats{});
|
||||
@@ -590,48 +730,116 @@ void DownloadTaskState::seg_finished(std::uint32_t seg_idx, Result<net::Transfer
|
||||
}
|
||||
return done();
|
||||
}
|
||||
if (assembling) {
|
||||
// The file is fully received; this worker was cancelled so its buffered tail lands
|
||||
// on disk. advance() has already counted these bytes; the flush makes them durable.
|
||||
if (w->buf) {
|
||||
if (auto f = w->buf->flush(); !f.has_value()) {
|
||||
release_slot();
|
||||
fail_locked(std::move(f).error());
|
||||
return done();
|
||||
}
|
||||
}
|
||||
if (pending_action != PendingAction::none) {
|
||||
// A sibling already decided the task is finishing (verify / fail / auto-pause /
|
||||
// demote); this worker's own outcome no longer matters. Drain it like every other
|
||||
// exit path: flush its buffer so segment_completed() stays true to disk, then hand
|
||||
// off to whichever worker finds the map empty.
|
||||
if (w->buf)
|
||||
(void)w->buf->flush(); // best-effort: we're already tearing down for another
|
||||
// reason, and the pending action doesn't depend on
|
||||
// this segment reaching any particular state.
|
||||
seg->advance(seg_idx, w->base_completed + w->recv);
|
||||
release_slot();
|
||||
if (workers.empty())
|
||||
if (workers.empty()) {
|
||||
PendingAction action = std::exchange(pending_action, PendingAction::none);
|
||||
ErrorInfo e = pending_error.value_or(ErrorInfo(Error::internal, ""));
|
||||
bool auth = pending_auth, decision = pending_decision;
|
||||
switch (action) {
|
||||
case PendingAction::verify:
|
||||
begin_verify_locked();
|
||||
break;
|
||||
case PendingAction::fail:
|
||||
fail_locked(e);
|
||||
break;
|
||||
case PendingAction::auto_pause:
|
||||
auto_pause_locked(e, auth, decision);
|
||||
break;
|
||||
case PendingAction::demote:
|
||||
demote_to_single_segment_locked();
|
||||
break;
|
||||
case PendingAction::none:
|
||||
break;
|
||||
}
|
||||
}
|
||||
return done();
|
||||
}
|
||||
if (w->needs_auth) {
|
||||
cancel_all_transfers_locked();
|
||||
release_slot();
|
||||
auto_pause_locked(ErrorInfo(Error::auth_required, "401/407", w->http_status), true, false);
|
||||
return done();
|
||||
}
|
||||
if (w->wrong_status) {
|
||||
cancel_all_transfers_locked();
|
||||
release_slot();
|
||||
auto_pause_locked(ErrorInfo(Error::server_file_changed, "200 where 206 expected"), false,
|
||||
true);
|
||||
// A 200 where 206 was expected is ambiguous: the file really changed (ask, don't
|
||||
// corrupt — docs/04 §5), or the server just stopped honouring Range for this
|
||||
// connection while it's still the same file (docs/04 §7: demote to 1 segment and
|
||||
// continue). ETag/Last-Modified from the 200 itself, compared against what the
|
||||
// probe recorded, is the only signal that tells them apart. Prefer ETag strictly
|
||||
// when both sides have one — same rule If-Range itself uses — and only fall back to
|
||||
// Last-Modified when there's no ETag to compare; a coarse (often second-resolution)
|
||||
// Last-Modified that happens to match is weak evidence next to a mismatching ETag.
|
||||
bool same_file;
|
||||
if (!probe.etag.empty() && !w->resp_etag.empty())
|
||||
same_file = probe.etag == w->resp_etag;
|
||||
else if (!probe.last_modified.empty() && !w->resp_last_modified.empty())
|
||||
same_file = probe.last_modified == w->resp_last_modified;
|
||||
else
|
||||
same_file = false; // no validator to compare -> can't prove it, ask
|
||||
if (same_file) {
|
||||
demote_to_single_segment_locked();
|
||||
} else {
|
||||
auto_pause_locked(ErrorInfo(Error::server_file_changed, "200 where 206 expected"),
|
||||
false, true);
|
||||
}
|
||||
return done();
|
||||
}
|
||||
if (w->flush_error) {
|
||||
ErrorInfo e = *w->flush_error;
|
||||
release_slot();
|
||||
if (e.code == Error::disk_full) {
|
||||
cancel_all_transfers_locked();
|
||||
auto_pause_locked(e, false, false);
|
||||
} else {
|
||||
fail_locked(e);
|
||||
}
|
||||
return done();
|
||||
}
|
||||
if (w->range_bad)
|
||||
r = Result<net::TransferStats>(ErrorInfo(Error::range_not_satisfiable, "416"));
|
||||
if (w->range_bad) {
|
||||
// 416 mid-download means our range metadata is stale (docs/04 §7): re-probe and
|
||||
// re-split rather than retrying the same now-invalid range until it exhausts.
|
||||
release_slot();
|
||||
auto_pause_locked(ErrorInfo(Error::range_not_satisfiable, "416"), false, true);
|
||||
return done();
|
||||
}
|
||||
if (w->forbidden) {
|
||||
// docs/04 §7: "403 after redirect: retry once with the original referrer -- many
|
||||
// CDNs require it." Bare/foreign Referer is the common cause; origin_of() rebuilds
|
||||
// it from the (possibly redirected) URL the response actually came from. Exactly
|
||||
// once per task, not a backoff series -- a second 403 with a same-origin Referer
|
||||
// already set isn't something another blind retry can fix (a private/expired
|
||||
// resource, an expiring signed URL past its window, ...). That's not necessarily
|
||||
// terminal, though: ask (same "ask, don't just fail outright" shape as
|
||||
// wrong_status/range_bad above) rather than fail_locked() outright, specifically
|
||||
// so DownloadHandle::refresh_url() -- do_refresh_url() is a no-op once the task is
|
||||
// terminal -- stays usable for the case tools/testserver's README pairs it with:
|
||||
// a caller that gets a fresh signed URL and hands it back.
|
||||
release_slot();
|
||||
if (!referrer_retried) {
|
||||
referrer_retried = true;
|
||||
effective_referrer = origin_of(current_url());
|
||||
seg->set_segment_state(seg_idx, segment::SegState::stalled);
|
||||
auto wp = weak_from_this();
|
||||
host.schedule(std::chrono::steady_clock::now(), [wp, seg_idx] {
|
||||
if (auto s = wp.lock())
|
||||
s->retry_worker(seg_idx);
|
||||
});
|
||||
if (workers.empty())
|
||||
transition(EngineState::retry_wait, std::nullopt);
|
||||
} else {
|
||||
auto_pause_locked(ErrorInfo(Error::forbidden, "403", w->http_status), false, true);
|
||||
}
|
||||
return done();
|
||||
}
|
||||
|
||||
if (!r.has_value()) {
|
||||
ErrorInfo e = std::move(r).error();
|
||||
@@ -695,17 +903,8 @@ void DownloadTaskState::seg_finished(std::uint32_t seg_idx, Result<net::Transfer
|
||||
else
|
||||
release_slot();
|
||||
|
||||
if (seg->all_complete()) {
|
||||
if (workers.empty()) {
|
||||
begin_verify_locked();
|
||||
} else {
|
||||
// Byte counters are satisfied, but other workers are still live and their
|
||||
// tails may only be in their buffers. Cancel them; each one's seg_finished
|
||||
// (this thread, once its curl worker has truly stopped) flushes via the
|
||||
// `assembling` branch, and the last starts verification.
|
||||
start_assembly_locked();
|
||||
}
|
||||
}
|
||||
if (seg->all_complete())
|
||||
begin_verify_locked(); // drain-aware: defers if other workers are still live
|
||||
return done();
|
||||
}
|
||||
|
||||
@@ -713,7 +912,7 @@ void DownloadTaskState::retry_worker(std::uint32_t seg_idx) {
|
||||
{
|
||||
std::unique_lock lk(mu);
|
||||
if (retired.load() || is_terminal(state) || pause_requested || cancel_requested ||
|
||||
assembling || !seg)
|
||||
pending_action != PendingAction::none || !seg)
|
||||
return;
|
||||
if (workers.count(seg_idx))
|
||||
return;
|
||||
@@ -729,6 +928,10 @@ void DownloadTaskState::retry_worker(std::uint32_t seg_idx) {
|
||||
}
|
||||
|
||||
void DownloadTaskState::begin_verify_locked() {
|
||||
if (!workers.empty()) {
|
||||
begin_drain_locked(PendingAction::verify, ErrorInfo(Error::internal, ""), false, false);
|
||||
return;
|
||||
}
|
||||
transition(EngineState::assembling, std::nullopt);
|
||||
transition(EngineState::verifying, std::nullopt);
|
||||
(void)file->sync();
|
||||
@@ -778,7 +981,10 @@ void DownloadTaskState::begin_verify_locked() {
|
||||
}
|
||||
|
||||
void DownloadTaskState::fail_locked(ErrorInfo e) {
|
||||
cancel_all_transfers_locked();
|
||||
if (!workers.empty()) {
|
||||
begin_drain_locked(PendingAction::fail, std::move(e), false, false);
|
||||
return;
|
||||
}
|
||||
if (file)
|
||||
(void)file->close();
|
||||
if (seg)
|
||||
@@ -800,6 +1006,10 @@ void DownloadTaskState::fail_locked(ErrorInfo e) {
|
||||
}
|
||||
|
||||
void DownloadTaskState::auto_pause_locked(ErrorInfo e, bool auth, bool decision) {
|
||||
if (!workers.empty()) {
|
||||
begin_drain_locked(PendingAction::auto_pause, std::move(e), auth, decision);
|
||||
return;
|
||||
}
|
||||
awaiting_auth = auth;
|
||||
awaiting_decision = decision;
|
||||
if (file)
|
||||
@@ -832,26 +1042,45 @@ void DownloadTaskState::auto_pause_locked(ErrorInfo e, bool auth, bool decision)
|
||||
}
|
||||
}
|
||||
|
||||
void DownloadTaskState::cancel_all_transfers_locked() {
|
||||
std::unique_lock wl(workers_mu);
|
||||
for (auto &[idx, w] : workers)
|
||||
w->transfer.cancel();
|
||||
workers.clear();
|
||||
}
|
||||
|
||||
// Every byte is received but some workers are still live; their buffered tails would be
|
||||
// lost if we dropped them here (seg_data() runs append() without workers_mu, so we cannot
|
||||
// safely flush another segment's buffer from under it). Just cancel them and let each
|
||||
// worker's own seg_finished drain it through the `assembling` branch once its curl worker
|
||||
// has stopped.
|
||||
void DownloadTaskState::start_assembly_locked() {
|
||||
assembling = true;
|
||||
transition(EngineState::assembling, std::nullopt);
|
||||
// Cancel every live worker and remember what to do once they've all drained through
|
||||
// seg_finished's PendingAction branch (see the enum's comment). Never clears `workers`
|
||||
// itself — each worker removes itself, flushed, when its own transfer actually completes.
|
||||
void DownloadTaskState::begin_drain_locked(PendingAction action, ErrorInfo e, bool auth,
|
||||
bool decision) {
|
||||
pending_action = action;
|
||||
pending_error = std::move(e);
|
||||
pending_auth = auth;
|
||||
pending_decision = decision;
|
||||
std::shared_lock wl(workers_mu);
|
||||
for (auto &[idx, w] : workers)
|
||||
w->transfer.cancel();
|
||||
}
|
||||
|
||||
// The 200-where-206-expected we just saw carried the same ETag/Last-Modified the probe
|
||||
// recorded: same file, the server (or this connection) just doesn't honour Range. Rebuild
|
||||
// as a single non-resumable segment covering the whole file and keep going with a plain
|
||||
// GET. It re-transfers bytes we may already have — there's no way to ask a Range-blind
|
||||
// server for a suffix — but it never truncates or discards what's on disk, and a source
|
||||
// that hasn't changed serves identical bytes, so the result is still byte-correct.
|
||||
void DownloadTaskState::demote_to_single_segment_locked() {
|
||||
if (!workers.empty()) {
|
||||
begin_drain_locked(PendingAction::demote, ErrorInfo(Error::internal, ""), false, false);
|
||||
return;
|
||||
}
|
||||
resumable = false;
|
||||
seg = std::make_unique<segment::Segmenter>(total_size.value_or(0), 1, /*resumable=*/false,
|
||||
host.config().min_segment_bytes);
|
||||
transition(EngineState::connecting, std::nullopt);
|
||||
if (registered) {
|
||||
// set_want() alone is not enough: if the demoted target happens to equal what it
|
||||
// was before the rebuild (e.g. this was already the last live segment), it's a
|
||||
// no-op and the async budget callback never fires. Drive slot assignment directly.
|
||||
slot_target = want_slots();
|
||||
host.budget().set_want(id, slot_target);
|
||||
fill_slots_locked();
|
||||
}
|
||||
}
|
||||
|
||||
void DownloadTaskState::finalize_cancel_locked() {
|
||||
if (file)
|
||||
(void)file->close();
|
||||
@@ -915,10 +1144,11 @@ void DownloadTaskState::emit_progress_if_due() {
|
||||
std::shared_lock lk(workers_mu);
|
||||
double agg = 0;
|
||||
for (auto &[idx, w] : workers) {
|
||||
agg += w->speed_bps;
|
||||
double speed = w->speed_bps.load(std::memory_order_relaxed);
|
||||
agg += speed;
|
||||
SegmentProgress sp;
|
||||
sp.index = idx;
|
||||
sp.speed_bps = static_cast<std::uint64_t>(w->speed_bps);
|
||||
sp.speed_bps = static_cast<std::uint64_t>(speed);
|
||||
p.segments.push_back(sp);
|
||||
}
|
||||
p.speed_bps = static_cast<std::uint64_t>(agg);
|
||||
@@ -1025,7 +1255,10 @@ void DownloadTaskState::do_decide(Decision d) {
|
||||
return;
|
||||
awaiting_decision = false;
|
||||
if (d == Decision::abort) {
|
||||
fail_locked(ErrorInfo(Error::server_file_changed, "user aborted"));
|
||||
// Surface the reason the decision was actually asked for (range_metadata_stale
|
||||
// sets last_error to range_not_satisfiable, server_file_changed to itself), not
|
||||
// a hardcoded label that would misreport a 416 as a changed file.
|
||||
fail_locked(last_error.value_or(ErrorInfo(Error::server_file_changed, "user aborted")));
|
||||
} else {
|
||||
if (d == Decision::restart) {
|
||||
::unlink(part_path.c_str());
|
||||
@@ -1066,6 +1299,7 @@ void DownloadTaskState::do_refresh_url(std::string url, std::vector<net::HeaderF
|
||||
net::ProbeRequest pr;
|
||||
pr.url = spec.url;
|
||||
pr.headers = spec.headers;
|
||||
pr.referrer = effective_referrer;
|
||||
pr.auth = spec.auth;
|
||||
pr.proxy = spec.proxy;
|
||||
host.probe(std::move(pr), [wp](Result<net::ProbeResult> r) {
|
||||
@@ -1075,10 +1309,43 @@ void DownloadTaskState::do_refresh_url(std::string url, std::vector<net::HeaderF
|
||||
std::unique_lock lk(s->mu);
|
||||
if (s->retired.load() || is_terminal(s->state))
|
||||
return;
|
||||
if (r.has_value()) {
|
||||
if (!r.has_value()) {
|
||||
lk.unlock();
|
||||
s->flush_deferred();
|
||||
return; // still paused; the caller can retry refresh_url() or decide()
|
||||
}
|
||||
if (!s->have_probe) {
|
||||
// The task's *first* probe never succeeded (e.g. this session's own
|
||||
// expiring-signed-url path: 403, one referrer retry, still 403 -> ask rather
|
||||
// than fail outright -- see on_probe_result() -- specifically so this branch
|
||||
// exists to recover it). finish_probe_locked() is what actually registers the
|
||||
// task with the budget and builds its Segmenter; nothing downstream of a
|
||||
// partial field copy would ever start a worker without it.
|
||||
s->probe = std::move(r).value();
|
||||
s->have_probe = true;
|
||||
s->awaiting_auth = false;
|
||||
s->awaiting_decision = false;
|
||||
s->finish_probe_locked();
|
||||
lk.unlock();
|
||||
s->flush_deferred();
|
||||
return;
|
||||
}
|
||||
s->probe.effective_url = r.value().effective_url;
|
||||
s->probe.etag = r.value().etag;
|
||||
s->probe.last_modified = r.value().last_modified;
|
||||
// refresh_url()'s own contract is "on a live OR PAUSED task, without losing
|
||||
// progress" -- distinct from do_decide(restart), which discards progress. A task
|
||||
// can be paused here for any of three reasons (a plain user pause, awaiting_auth,
|
||||
// or awaiting_decision -- e.g. this session's own 403-after-referrer-retry path,
|
||||
// or the pre-existing wrong_status/range_bad ones); apply_slot_target()'s guard
|
||||
// blocks on awaiting_auth/awaiting_decision specifically, so leaving either set
|
||||
// would have set_want() below recompute a target that nothing ever acts on --
|
||||
// the caller's new URL re-probed successfully and then the task just sat there.
|
||||
// Clear both and leave `paused` the same way do_decide(restart) does.
|
||||
if (s->state == EngineState::paused) {
|
||||
s->awaiting_auth = false;
|
||||
s->awaiting_decision = false;
|
||||
s->transition(EngineState::connecting, std::nullopt);
|
||||
}
|
||||
if (s->registered)
|
||||
s->host.budget().set_want(s->id, s->want_slots());
|
||||
@@ -1088,12 +1355,24 @@ void DownloadTaskState::do_refresh_url(std::string url, std::vector<net::HeaderF
|
||||
}
|
||||
|
||||
void DownloadTaskState::quiesce() {
|
||||
std::lock_guard lk(mu);
|
||||
std::unique_lock lk(mu);
|
||||
retired.store(true);
|
||||
std::unique_lock wl(workers_mu);
|
||||
{
|
||||
std::shared_lock wl(workers_mu);
|
||||
for (auto &[idx, w] : workers)
|
||||
w->transfer.cancel();
|
||||
workers.clear();
|
||||
}
|
||||
// Do not clear `workers` here: transfer.cancel() only requests the HttpClient worker
|
||||
// thread stop the transfer, asynchronously -- it does not wait for that to happen. A
|
||||
// worker's SegWorker (and its WriteBuffer) may still be in active use by a curl write
|
||||
// callback running on that other thread right now. Clearing the map out from under it
|
||||
// was a real, ASan-caught heap-use-after-free (ring buffer freed here while
|
||||
// SparseFile::write_at() on the HttpClient worker thread was still writing through it).
|
||||
// Every worker removes and flushes itself, safely, via seg_finished once HttpClient
|
||||
// actually confirms the transfer has stopped (same path every other exit uses; see the
|
||||
// `retired` branch there) -- just wait for that to happen for all of them. Bounded by
|
||||
// however long a cancelled curl transfer takes to unwind, not user-controllable.
|
||||
workers_drained_cv.wait(lk, [this] { return workers.empty(); });
|
||||
}
|
||||
|
||||
EngineState DownloadTaskState::snapshot_state() {
|
||||
@@ -1104,6 +1383,26 @@ EngineState DownloadTaskState::snapshot_state() {
|
||||
Progress DownloadTaskState::snapshot_progress() {
|
||||
Progress p;
|
||||
std::lock_guard lk(mu);
|
||||
// Per-segment instantaneous speed lives on the live SegWorker (seg_data's 0.5s-sampled
|
||||
// EMA, see the `speed_bps` update below) -- a segment with no live worker (idle,
|
||||
// paused, complete, failed) has no speed to report and stays at SegmentProgress's
|
||||
// default 0. Read every live worker's speed up front so the seg->snapshot() loop below
|
||||
// (which covers *every* segment, not just live ones -- unlike emit_progress_if_due's
|
||||
// push-callback version, which only ever reports the segments it currently has
|
||||
// workers for) can look each one up by index.
|
||||
double agg_speed = 0;
|
||||
std::unordered_map<std::uint32_t, double> worker_speed;
|
||||
{
|
||||
std::shared_lock wl(workers_mu);
|
||||
worker_speed.reserve(workers.size());
|
||||
for (auto &[idx, w] : workers) {
|
||||
double speed = w->speed_bps.load(std::memory_order_relaxed);
|
||||
worker_speed.emplace(idx, speed);
|
||||
agg_speed += speed;
|
||||
}
|
||||
p.effective_segments = static_cast<std::uint32_t>(workers.size());
|
||||
}
|
||||
p.speed_bps = static_cast<std::uint64_t>(agg_speed);
|
||||
if (seg) {
|
||||
p.downloaded = seg->downloaded();
|
||||
for (auto &v : seg->snapshot()) {
|
||||
@@ -1113,15 +1412,15 @@ Progress DownloadTaskState::snapshot_progress() {
|
||||
sp.end = v.end;
|
||||
sp.completed = v.completed;
|
||||
sp.state = v.state;
|
||||
if (auto it = worker_speed.find(v.index); it != worker_speed.end())
|
||||
sp.speed_bps = static_cast<std::uint64_t>(it->second);
|
||||
p.segments.push_back(sp);
|
||||
}
|
||||
}
|
||||
p.total = total_size;
|
||||
{
|
||||
std::shared_lock wl(workers_mu);
|
||||
p.effective_segments = static_cast<std::uint32_t>(workers.size());
|
||||
}
|
||||
p.effective_buffer_bytes = effective_buffer;
|
||||
if (p.speed_bps > 0 && total_size && *total_size > p.downloaded)
|
||||
p.eta_seconds = static_cast<std::uint32_t>((*total_size - p.downloaded) / p.speed_bps);
|
||||
return p;
|
||||
}
|
||||
|
||||
|
||||
@@ -55,3 +55,7 @@ if(NOT EXISTS ${_testserver})
|
||||
message(STATUS "veloxcore: tools/testserver not present; net integration tests will "
|
||||
"skip their server-backed cases.")
|
||||
endif()
|
||||
|
||||
vdm_add_test(veloxcore_rules_filename_test rules/filename_test.cpp)
|
||||
vdm_add_test(veloxcore_rules_collision_test rules/collision_test.cpp)
|
||||
vdm_add_test(veloxcore_rules_match_test rules/match_test.cpp)
|
||||
|
||||
@@ -28,7 +28,14 @@ namespace vdm::testing {
|
||||
|
||||
class TestServer {
|
||||
public:
|
||||
TestServer() {
|
||||
TestServer() : TestServer(1.0) {}
|
||||
|
||||
// loris_seconds overrides the dribble duration slow-loris mode uses (default matches the
|
||||
// no-arg ctor's long-standing 1s). A test that needs curl's stall detector
|
||||
// (CURLOPT_LOW_SPEED_TIME, hardcoded to 30s in download_task.cpp) to actually fire needs a
|
||||
// dribble that outlasts that threshold, not the short one every other test relies on to
|
||||
// keep runtime down.
|
||||
explicit TestServer(double loris_seconds) {
|
||||
const char *script = VDM_TESTSERVER_PY;
|
||||
if (!script || !*script || ::access(script, R_OK) != 0)
|
||||
return;
|
||||
@@ -50,8 +57,9 @@ class TestServer {
|
||||
int devnull = ::open("/dev/null", O_WRONLY);
|
||||
if (devnull >= 0)
|
||||
::dup2(devnull, STDERR_FILENO);
|
||||
std::string loris_str = std::to_string(loris_seconds);
|
||||
::execlp("python3", "python3", script, "--port", "0", "--seed", "9", "--loris-seconds",
|
||||
"1", "--throttle-bps", "131072", static_cast<char *>(nullptr));
|
||||
loris_str.c_str(), "--throttle-bps", "131072", static_cast<char *>(nullptr));
|
||||
::_exit(127);
|
||||
}
|
||||
::close(pipefd[1]);
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
#include "vdm/rules/collision.hpp"
|
||||
|
||||
#include <functional>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#include "vtest.hpp"
|
||||
|
||||
using vdm::rules::CollisionPolicy;
|
||||
using vdm::rules::resolve_collision;
|
||||
|
||||
namespace {
|
||||
std::function<bool(std::string_view)> exists_in(const std::set<std::string> &names) {
|
||||
return [&names](std::string_view s) { return names.count(std::string(s)) > 0; };
|
||||
}
|
||||
} // namespace
|
||||
|
||||
VT_TEST(collision_no_collision_returns_desired) {
|
||||
std::set<std::string> existing = {"other.txt"};
|
||||
VT_CHECK_EQ(resolve_collision("file.txt", exists_in(existing)), std::string("file.txt"));
|
||||
}
|
||||
|
||||
VT_TEST(collision_renames_on_conflict) {
|
||||
std::set<std::string> existing = {"file.txt"};
|
||||
VT_CHECK_EQ(resolve_collision("file.txt", exists_in(existing)), std::string("file (1).txt"));
|
||||
}
|
||||
|
||||
VT_TEST(collision_finds_first_free_slot) {
|
||||
std::set<std::string> existing = {"file.txt", "file (1).txt", "file (2).txt"};
|
||||
VT_CHECK_EQ(resolve_collision("file.txt", exists_in(existing)), std::string("file (3).txt"));
|
||||
}
|
||||
|
||||
VT_TEST(collision_no_extension) {
|
||||
std::set<std::string> existing = {"README"};
|
||||
VT_CHECK_EQ(resolve_collision("README", exists_in(existing)), std::string("README (1)"));
|
||||
}
|
||||
|
||||
VT_TEST(collision_dotfile_treated_as_no_extension) {
|
||||
std::set<std::string> existing = {".gitignore"};
|
||||
VT_CHECK_EQ(resolve_collision(".gitignore", exists_in(existing)),
|
||||
std::string(".gitignore (1)"));
|
||||
}
|
||||
|
||||
VT_TEST(collision_overwrite_policy_ignores_existence) {
|
||||
std::set<std::string> existing = {"file.txt"};
|
||||
VT_CHECK_EQ(resolve_collision("file.txt", exists_in(existing), CollisionPolicy::overwrite),
|
||||
std::string("file.txt"));
|
||||
}
|
||||
|
||||
VT_TEST(collision_gives_up_after_max_attempts_without_fabricating) {
|
||||
auto always_exists = [](std::string_view) { return true; };
|
||||
auto out = resolve_collision("file.txt", always_exists, CollisionPolicy::rename, 3);
|
||||
VT_CHECK_EQ(out, std::string("file (3).txt")); // last attempted, still colliding
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
#include "vdm/rules/filename.hpp"
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "vtest.hpp"
|
||||
|
||||
using vdm::rules::sanitize_filename;
|
||||
|
||||
VT_TEST(filename_passthrough_when_already_clean) {
|
||||
VT_CHECK_EQ(sanitize_filename("report.pdf"), std::string("report.pdf"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_strips_path_separators) {
|
||||
VT_CHECK_EQ(sanitize_filename("a/b\\c.txt"), std::string("abc.txt"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_collapses_dotdot_after_separator_strip) {
|
||||
// "../../etc/passwd" -> separators stripped, then the resulting ".." runs collapse to
|
||||
// a single '.', which strip-trailing-dot then removes entirely.
|
||||
auto out = sanitize_filename("../../etc/passwd");
|
||||
VT_CHECK(out.find("..") == std::string::npos);
|
||||
}
|
||||
|
||||
VT_TEST(filename_strips_control_bytes) {
|
||||
std::string raw = "bad";
|
||||
raw.push_back('\0');
|
||||
raw += "name.txt";
|
||||
auto out = sanitize_filename(raw);
|
||||
VT_CHECK_EQ(out, std::string("badname.txt"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_replaces_ntfs_illegal_chars) {
|
||||
VT_CHECK_EQ(sanitize_filename("a<b>c:d\"e|f?g*h.txt"), std::string("a_b_c_d_e_f_g_h.txt"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_strips_trailing_dot_and_space) {
|
||||
VT_CHECK_EQ(sanitize_filename("name. "), std::string("name"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_empty_falls_back_to_download) {
|
||||
VT_CHECK_EQ(sanitize_filename(""), std::string("download"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_all_stripped_falls_back_to_download) {
|
||||
VT_CHECK_EQ(sanitize_filename("/\\"), std::string("download"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_reserved_device_name_bare) {
|
||||
VT_CHECK_EQ(sanitize_filename("CON"), std::string("CON_"));
|
||||
VT_CHECK_EQ(sanitize_filename("con"), std::string("con_"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_reserved_device_name_with_extension) {
|
||||
VT_CHECK_EQ(sanitize_filename("NUL.txt"), std::string("NUL_.txt"));
|
||||
VT_CHECK_EQ(sanitize_filename("com3.tar.gz"), std::string("com3_.tar.gz"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_reserved_device_name_not_a_false_positive) {
|
||||
// "CONTEST" is not "CON" — must not get mangled.
|
||||
VT_CHECK_EQ(sanitize_filename("CONTEST.txt"), std::string("CONTEST.txt"));
|
||||
VT_CHECK_EQ(sanitize_filename("COM99.txt"), std::string("COM99.txt")); // not COM1-9
|
||||
}
|
||||
|
||||
VT_TEST(filename_truncates_long_name_keeping_extension) {
|
||||
std::string stem(500, 'a');
|
||||
auto out = sanitize_filename(stem + ".txt", 255);
|
||||
VT_CHECK(out.size() <= 255);
|
||||
VT_CHECK(out.ends_with(".txt"));
|
||||
}
|
||||
|
||||
VT_TEST(filename_truncation_is_utf8_safe) {
|
||||
// Each "é" is 2 bytes (C3 A9); a 5-byte budget can fit 2 whole codepoints (4 bytes) but
|
||||
// not a 3rd (needs 6) -- an unguarded byte-length cut at 5 would split the 3rd
|
||||
// codepoint's C3 from its A9, leaving a dangling lead byte.
|
||||
std::string stem;
|
||||
for (int i = 0; i < 20; ++i) stem += "\xC3\xA9";
|
||||
auto out = sanitize_filename(stem, 5);
|
||||
VT_CHECK_EQ(out, std::string("\xC3\xA9\xC3\xA9")); // 2 whole codepoints, 4 bytes
|
||||
}
|
||||
|
||||
VT_TEST(filename_preserves_non_ascii) {
|
||||
VT_CHECK_EQ(sanitize_filename("caf\xC3\xA9.pdf"), std::string("caf\xC3\xA9.pdf"));
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
#include "vdm/rules/match.hpp"
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "vtest.hpp"
|
||||
|
||||
using namespace vdm::rules;
|
||||
|
||||
namespace {
|
||||
|
||||
Rule make_rule(std::string id, std::int64_t priority, RuleMatch m, RuleAction a,
|
||||
bool enabled = true) {
|
||||
Rule r;
|
||||
r.rule_id = std::move(id);
|
||||
r.priority = priority;
|
||||
r.enabled = enabled;
|
||||
r.match = std::move(m);
|
||||
r.action = std::move(a);
|
||||
return r;
|
||||
}
|
||||
|
||||
// -Wmissing-field-initializers (part of -Wextra) flags a designated-initializer list that
|
||||
// skips a member, even one this repo's designated-init style would normally leave
|
||||
// implicit -- these small builders keep the tests below readable without tripping it.
|
||||
RuleAction action_with_category(std::string id) {
|
||||
RuleAction a;
|
||||
a.category_id = std::move(id);
|
||||
return a;
|
||||
}
|
||||
|
||||
MatchInput input_with_extension(std::string ext) {
|
||||
MatchInput in;
|
||||
in.extension = std::move(ext);
|
||||
return in;
|
||||
}
|
||||
|
||||
MatchInput input_with_host(std::string host) {
|
||||
MatchInput in;
|
||||
in.host = std::move(host);
|
||||
return in;
|
||||
}
|
||||
|
||||
MatchInput input_with_size(std::optional<std::uint64_t> size) {
|
||||
MatchInput in;
|
||||
in.size_bytes = size;
|
||||
return in;
|
||||
}
|
||||
|
||||
MatchInput input_with_extension_and_host(std::string ext, std::string host) {
|
||||
MatchInput in;
|
||||
in.extension = std::move(ext);
|
||||
in.host = std::move(host);
|
||||
return in;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// --- glob_match ---------------------------------------------------------------------
|
||||
|
||||
VT_TEST(glob_exact_match) {
|
||||
VT_CHECK(glob_match("example.com", "example.com"));
|
||||
VT_CHECK(!glob_match("example.com", "example.org"));
|
||||
}
|
||||
|
||||
VT_TEST(glob_star_suffix) {
|
||||
VT_CHECK(glob_match("*.example.com", "cdn.example.com"));
|
||||
VT_CHECK(glob_match("*.example.com", "a.b.example.com"));
|
||||
VT_CHECK(!glob_match("*.example.com", "example.com")); // no room for the literal '.'
|
||||
}
|
||||
|
||||
VT_TEST(glob_star_matches_empty) {
|
||||
VT_CHECK(glob_match("file*.zip", "file.zip"));
|
||||
VT_CHECK(glob_match("file*.zip", "file123.zip"));
|
||||
}
|
||||
|
||||
VT_TEST(glob_question_mark) {
|
||||
VT_CHECK(glob_match("file?.txt", "file1.txt"));
|
||||
VT_CHECK(!glob_match("file?.txt", "file.txt"));
|
||||
VT_CHECK(!glob_match("file?.txt", "file12.txt"));
|
||||
}
|
||||
|
||||
VT_TEST(glob_case_insensitive) {
|
||||
VT_CHECK(glob_match("*.EXAMPLE.com", "cdn.example.COM"));
|
||||
}
|
||||
|
||||
VT_TEST(glob_multiple_stars) {
|
||||
VT_CHECK(glob_match("*foo*bar*", "xxfooyybarzz"));
|
||||
VT_CHECK(!glob_match("*foo*bar*", "xxbarzzfooyy")); // order matters
|
||||
}
|
||||
|
||||
VT_TEST(glob_pathological_stars_terminate) {
|
||||
// A pattern of nothing but '*' against a long text must not blow up (bounded work).
|
||||
std::string pattern(50, '*');
|
||||
std::string text(10000, 'x');
|
||||
VT_CHECK(glob_match(pattern, text));
|
||||
}
|
||||
|
||||
// --- match_rules ---------------------------------------------------------------------
|
||||
|
||||
VT_TEST(match_empty_table_yields_nullopt) {
|
||||
VT_CHECK(!match_rules({}, input_with_extension("zip")).has_value());
|
||||
}
|
||||
|
||||
VT_TEST(match_no_clause_matches_everything) {
|
||||
auto rules = {make_rule("r1", 0, RuleMatch{}, action_with_category("default"))};
|
||||
auto r = match_rules(rules, input_with_extension("anything"));
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(*r->category_id, std::string("default"));
|
||||
}
|
||||
|
||||
VT_TEST(match_by_extension_case_insensitive) {
|
||||
RuleMatch m;
|
||||
m.extensions = std::vector<std::string>{"zip", "rar"};
|
||||
auto rules = {make_rule("r1", 0, m, action_with_category("archives"))};
|
||||
VT_CHECK(match_rules(rules, input_with_extension("ZIP")).has_value());
|
||||
VT_CHECK(!match_rules(rules, input_with_extension("txt")).has_value());
|
||||
}
|
||||
|
||||
VT_TEST(match_priority_order_lower_wins) {
|
||||
std::vector<Rule> rules = {
|
||||
make_rule("hi", 10, RuleMatch{}, action_with_category("first")),
|
||||
make_rule("lo", 0, RuleMatch{}, action_with_category("second")),
|
||||
};
|
||||
auto r = match_rules(rules, MatchInput{});
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(*r->category_id, std::string("second")); // priority 0 runs first
|
||||
}
|
||||
|
||||
VT_TEST(match_ties_keep_table_order) {
|
||||
std::vector<Rule> rules = {
|
||||
make_rule("a", 5, RuleMatch{}, action_with_category("first")),
|
||||
make_rule("b", 5, RuleMatch{}, action_with_category("second")),
|
||||
};
|
||||
auto r = match_rules(rules, MatchInput{});
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(*r->category_id, std::string("first"));
|
||||
}
|
||||
|
||||
VT_TEST(match_skips_disabled_rules) {
|
||||
std::vector<Rule> rules = {
|
||||
make_rule("a", 0, RuleMatch{}, action_with_category("disabled"), /*enabled=*/false),
|
||||
make_rule("b", 1, RuleMatch{}, action_with_category("enabled")),
|
||||
};
|
||||
auto r = match_rules(rules, MatchInput{});
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(*r->category_id, std::string("enabled"));
|
||||
}
|
||||
|
||||
VT_TEST(match_host_pattern) {
|
||||
RuleMatch m;
|
||||
m.host_pattern = "*.cdn.example.com";
|
||||
auto rules = {make_rule("r1", 0, m, action_with_category("cdn"))};
|
||||
VT_CHECK(match_rules(rules, input_with_host("a.cdn.example.com")).has_value());
|
||||
VT_CHECK(!match_rules(rules, input_with_host("example.com")).has_value());
|
||||
}
|
||||
|
||||
VT_TEST(match_size_bounds) {
|
||||
RuleMatch m;
|
||||
m.min_size_bytes = 1000;
|
||||
m.max_size_bytes = 2000;
|
||||
auto rules = {make_rule("r1", 0, m, action_with_category("midsize"))};
|
||||
VT_CHECK(match_rules(rules, input_with_size(1500)).has_value());
|
||||
VT_CHECK(!match_rules(rules, input_with_size(500)).has_value());
|
||||
VT_CHECK(!match_rules(rules, input_with_size(5000)).has_value());
|
||||
}
|
||||
|
||||
VT_TEST(match_size_clause_with_unknown_size_does_not_match) {
|
||||
RuleMatch m;
|
||||
m.min_size_bytes = 1000;
|
||||
auto rules = {make_rule("r1", 0, m, action_with_category("big"))};
|
||||
// size_bytes left absent (pre-probe) -- a size clause must not match speculatively.
|
||||
VT_CHECK(!match_rules(rules, MatchInput{}).has_value());
|
||||
}
|
||||
|
||||
VT_TEST(match_all_clauses_must_hold) {
|
||||
RuleMatch m;
|
||||
m.extensions = std::vector<std::string>{"iso"};
|
||||
m.host_pattern = "*.trusted.example";
|
||||
auto rules = {make_rule("r1", 0, m, action_with_category("isos"))};
|
||||
VT_CHECK(match_rules(rules, input_with_extension_and_host("iso", "mirror.trusted.example"))
|
||||
.has_value());
|
||||
// Extension matches but host doesn't -- must not match.
|
||||
VT_CHECK(!match_rules(rules, input_with_extension_and_host("iso", "evil.example"))
|
||||
.has_value());
|
||||
}
|
||||
|
||||
VT_TEST(match_falls_through_to_default_when_nothing_matches) {
|
||||
RuleMatch m;
|
||||
m.extensions = std::vector<std::string>{"exe"};
|
||||
auto rules = {make_rule("r1", 0, m, action_with_category("installers"))};
|
||||
VT_CHECK(!match_rules(rules, input_with_extension("pdf")).has_value());
|
||||
}
|
||||
@@ -2,7 +2,10 @@
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
@@ -20,17 +23,141 @@ TaskId tid(std::uint64_t v) {
|
||||
|
||||
// A test task that reacts to slot targets the way stage 8's download_task will: start
|
||||
// workers up to the target, release them when the target drops. Purely bookkeeping.
|
||||
//
|
||||
// Production's real equivalent (register_task()'s on_target lambda, download_task.cpp)
|
||||
// never calls back in synchronously -- it posts through host.schedule() (engine.cpp), so
|
||||
// apply_slot_target() never runs on the same call stack as whatever budget call triggered
|
||||
// it. wake_one_waiter_locked()'s wakeups, though, mean on_target() here *can* now be
|
||||
// reentered on the same thread (e.g. this task's own release_slot() call, deep in the grow
|
||||
// loop below, can cascade into waking a *different* task whose own release, in turn,
|
||||
// cascades back to this one) -- a plain std::mutex would self-deadlock on that, the way
|
||||
// FakeTask::mu almost did until this was written. A recursive_mutex plus coalescing the
|
||||
// reentrant call's target into `pending` (processed by the outer call's loop once its
|
||||
// current pass finishes) keeps this a faithful, deterministic stand-in without actually
|
||||
// needing real threads: no callback ever runs nested inside a still-running instance of
|
||||
// itself, and a burst of reentrant retargeting converges to the latest value instead of
|
||||
// each one doing its own redundant grow/shrink pass.
|
||||
struct FakeTask {
|
||||
SegmentBudget *budget = nullptr;
|
||||
TaskId id{};
|
||||
std::mutex mu;
|
||||
std::recursive_mutex mu;
|
||||
std::uint32_t workers = 0;
|
||||
std::uint32_t target = 0;
|
||||
bool running = false;
|
||||
std::optional<std::uint32_t> pending;
|
||||
|
||||
FakeTask() = default;
|
||||
FakeTask(SegmentBudget *b, TaskId i) : budget(b), id(i) {}
|
||||
|
||||
void on_target(std::uint32_t t) {
|
||||
std::lock_guard lk(mu);
|
||||
if (running) {
|
||||
pending = t; // reentrant call on this thread -- the running instance picks it up
|
||||
return;
|
||||
}
|
||||
running = true;
|
||||
std::uint32_t want = t;
|
||||
for (;;) {
|
||||
target = want;
|
||||
while (workers < target) {
|
||||
if (!budget->confirm_slot(id))
|
||||
break;
|
||||
++workers;
|
||||
}
|
||||
// over target -> yield the excess immediately (a real task waits for a
|
||||
// boundary)
|
||||
while (workers > target) {
|
||||
budget->release_slot(id);
|
||||
--workers;
|
||||
}
|
||||
if (!pending)
|
||||
break;
|
||||
want = *pending;
|
||||
pending.reset();
|
||||
}
|
||||
running = false;
|
||||
}
|
||||
std::uint32_t held() {
|
||||
std::lock_guard lk(mu);
|
||||
return workers;
|
||||
}
|
||||
};
|
||||
|
||||
// Mirrors Engine's real timer thread (engine.cpp: timer_loop) and how register_task()'s
|
||||
// on_target lambda actually reaches a task (download_task.cpp: host.schedule(), never a
|
||||
// direct call). One dedicated thread drains queued closures one at a time -- the
|
||||
// serialization that makes production immune to the cross-task deadlock risk FakeTask's
|
||||
// synchronous, whatever-thread-triggered-it delivery has under heavy *concurrent* driving
|
||||
// (budget_concurrent_confirm_release_stays_consistent, below, is the one test that
|
||||
// actually exercises this: multiple threads calling set_want() for *different* tasks at
|
||||
// once, each able to cascade into a wake for another task's callback -- two such cascades
|
||||
// landing on two different FakeTask mutexes in opposite orders on two different threads is
|
||||
// a real AB-BA deadlock a recursive_mutex alone doesn't prevent, since that only guards a
|
||||
// single thread against re-entering itself). Every other test in this file drives the
|
||||
// budget from one thread at a time, where that risk can't arise, so plain FakeTask is
|
||||
// still the simpler, sufficient double there.
|
||||
class TestTimer {
|
||||
public:
|
||||
TestTimer() {
|
||||
worker_ = std::jthread([this](std::stop_token st) { run(st); });
|
||||
}
|
||||
~TestTimer() {
|
||||
worker_.request_stop();
|
||||
cv_.notify_all();
|
||||
}
|
||||
void post(std::function<void()> fn) {
|
||||
{
|
||||
std::lock_guard lk(mu_);
|
||||
queue_.push_back(std::move(fn));
|
||||
}
|
||||
cv_.notify_one();
|
||||
}
|
||||
// Blocks until the queue is empty and nothing is mid-run -- what a test needs before
|
||||
// asserting on state this timer's closures mutate.
|
||||
void drain() {
|
||||
std::unique_lock lk(mu_);
|
||||
cv_done_.wait(lk, [&] { return queue_.empty() && !running_; });
|
||||
}
|
||||
|
||||
private:
|
||||
void run(std::stop_token st) {
|
||||
std::unique_lock lk(mu_);
|
||||
while (true) {
|
||||
cv_.wait(lk, st, [&] { return !queue_.empty(); });
|
||||
if (st.stop_requested())
|
||||
return;
|
||||
auto fn = std::move(queue_.front());
|
||||
queue_.erase(queue_.begin());
|
||||
running_ = true;
|
||||
lk.unlock();
|
||||
fn();
|
||||
lk.lock();
|
||||
running_ = false;
|
||||
if (queue_.empty())
|
||||
cv_done_.notify_all();
|
||||
}
|
||||
}
|
||||
std::mutex mu_;
|
||||
std::condition_variable_any cv_; // _any: wait() below takes a stop_token predicate
|
||||
std::condition_variable cv_done_;
|
||||
std::vector<std::function<void()>> queue_;
|
||||
bool running_ = false;
|
||||
std::jthread worker_;
|
||||
};
|
||||
|
||||
// Same grow/shrink logic as FakeTask, but on_target() only ever posts through a TestTimer
|
||||
// instead of running synchronously -- see the comment above TestTimer for why that's the
|
||||
// faithful model under concurrent driving.
|
||||
struct AsyncFakeTask {
|
||||
SegmentBudget *budget = nullptr;
|
||||
TestTimer *timer = nullptr;
|
||||
TaskId id{};
|
||||
std::mutex mu; // only the timer thread ever touches workers/target -- plain suffices
|
||||
std::uint32_t workers = 0;
|
||||
std::uint32_t target = 0;
|
||||
|
||||
void on_target(std::uint32_t t) {
|
||||
timer->post([this, t] {
|
||||
std::lock_guard lk(mu);
|
||||
target = t;
|
||||
while (workers < target) {
|
||||
@@ -38,11 +165,11 @@ struct FakeTask {
|
||||
break;
|
||||
++workers;
|
||||
}
|
||||
// over target -> yield the excess immediately (a real task waits for a boundary)
|
||||
while (workers > target) {
|
||||
budget->release_slot(id);
|
||||
--workers;
|
||||
}
|
||||
});
|
||||
}
|
||||
std::uint32_t held() {
|
||||
std::lock_guard lk(mu);
|
||||
@@ -235,12 +362,14 @@ VT_TEST(budget_on_changed_fires_on_starved_edge) {
|
||||
VT_TEST(budget_concurrent_confirm_release_stays_consistent) {
|
||||
SegmentBudget b({.max_active_segments = 16});
|
||||
constexpr int kTasks = 6;
|
||||
std::vector<std::unique_ptr<FakeTask>> ts;
|
||||
TestTimer timer;
|
||||
std::vector<std::unique_ptr<AsyncFakeTask>> ts;
|
||||
for (int i = 0; i < kTasks; ++i) {
|
||||
ts.push_back(std::make_unique<FakeTask>());
|
||||
ts.push_back(std::make_unique<AsyncFakeTask>());
|
||||
ts.back()->budget = &b;
|
||||
ts.back()->timer = &timer;
|
||||
ts.back()->id = tid(i + 1);
|
||||
FakeTask *ft = ts.back().get();
|
||||
AsyncFakeTask *ft = ts.back().get();
|
||||
b.register_task(ft->id, {.host = "h", .per_task_cap = 6, .resumable = true},
|
||||
[ft](std::uint32_t n) { ft->on_target(n); });
|
||||
}
|
||||
@@ -254,6 +383,7 @@ VT_TEST(budget_concurrent_confirm_release_stays_consistent) {
|
||||
drivers.clear(); // join
|
||||
for (auto &ft : ts)
|
||||
b.set_want(ft->id, 0);
|
||||
timer.drain(); // let every queued on_target actually run before asserting
|
||||
|
||||
// With everyone wanting nothing, the budget must be fully released.
|
||||
VT_CHECK_EQ(b.budget().active, 0u);
|
||||
@@ -262,3 +392,123 @@ VT_TEST(budget_concurrent_confirm_release_stays_consistent) {
|
||||
sum += b.segments_active(ft->id);
|
||||
VT_CHECK_EQ(sum, 0u);
|
||||
}
|
||||
|
||||
// --- wait-list wakeup: tools/bench heap-profile / load found a real task time out
|
||||
// waiting on a slot its own target already said it should have (core/docs/m7-baseline.md,
|
||||
// docs/adr/0012). Root cause: confirm_slot()'s engine-wide cap check (needed so active_
|
||||
// never exceeds max_active_segments -- a real over-admission bug, not just this liveness
|
||||
// gap) can deny a task whose target is already correct, when a *different* task is
|
||||
// legitimately still holding more than its own just-lowered target (yield is deferred to
|
||||
// a segment boundary, ADR 0011 A1). Nothing in the plain target-changed callback
|
||||
// mechanism ever revisits a task whose target didn't change -- it was already right. ---
|
||||
|
||||
namespace {
|
||||
|
||||
// Unlike FakeTask above, on_target() here only enqueues -- it never calls back into the
|
||||
// budget synchronously. This matches production exactly: register_task()'s on_target
|
||||
// lambda (download_task.cpp) posts through host.schedule() (engine.cpp), so
|
||||
// apply_slot_target() never runs on the same call stack as whatever budget call triggered
|
||||
// it. The test drives delivery explicitly (deliver_one()) instead of a background thread
|
||||
// so the race this test exists to force -- confirm_slot() denied before the task that's
|
||||
// over its target has processed its own shrink -- is deterministic, not a timing gamble.
|
||||
struct QueuedTask {
|
||||
SegmentBudget *budget = nullptr;
|
||||
TaskId id{};
|
||||
std::uint32_t workers = 0;
|
||||
std::uint32_t target = 0;
|
||||
std::vector<std::uint32_t> pending;
|
||||
|
||||
QueuedTask(SegmentBudget *b, TaskId i) : budget(b), id(i) {}
|
||||
|
||||
void on_target(std::uint32_t n) { pending.push_back(n); }
|
||||
|
||||
// Delivers the oldest queued target, applying it the way a real task's
|
||||
// apply_slot_target()/fill_slots_locked() would: try to grow to it (confirm_slot()
|
||||
// may deny), or shed down to it. Returns false (nothing to deliver) if the queue was
|
||||
// empty -- the condition VT_REQUIRE checks to prove a wakeup was actually queued.
|
||||
bool deliver_one() {
|
||||
if (pending.empty())
|
||||
return false;
|
||||
target = pending.front();
|
||||
pending.erase(pending.begin());
|
||||
while (workers < target) {
|
||||
if (!budget->confirm_slot(id))
|
||||
break;
|
||||
++workers;
|
||||
}
|
||||
while (workers > target) {
|
||||
budget->release_slot(id);
|
||||
--workers;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
VT_TEST(budget_release_wakes_a_denied_waiter) {
|
||||
// Sanity baseline: max_active=1, A (higher priority) holds it, B wants one too and is
|
||||
// fairly denied -- its target stays 0 while A outranks it and still wants its slot.
|
||||
// Once A stops wanting one, B's target rises and B is woken via the plain
|
||||
// target-changed path -- no wait-list needed for this simple case. The harder case
|
||||
// below is what actually needs it.
|
||||
SegmentBudget b({.max_active_segments = 1});
|
||||
QueuedTask A{&b, tid(1)}, B{&b, tid(2)};
|
||||
b.register_task(A.id, {.host = "h", .per_task_cap = 1, .resumable = true},
|
||||
[&](std::uint32_t n) { A.on_target(n); });
|
||||
b.set_task_order(std::vector<TaskId>{A.id, B.id});
|
||||
b.set_want(A.id, 1);
|
||||
VT_REQUIRE(A.deliver_one());
|
||||
VT_CHECK_EQ(A.workers, 1u);
|
||||
|
||||
// B registers and wants one too, but with A (higher priority) already holding the
|
||||
// only slot and still wanting it, B's fairly computed target stays 0 -- unchanged
|
||||
// from its just-registered value, so no callback is queued for it yet.
|
||||
b.register_task(B.id, {.host = "h", .per_task_cap = 1, .resumable = true},
|
||||
[&](std::uint32_t n) { B.on_target(n); });
|
||||
b.set_want(B.id, 1);
|
||||
VT_CHECK(!B.deliver_one());
|
||||
VT_CHECK_EQ(B.workers, 0u);
|
||||
|
||||
b.set_want(A.id, 0); // A is done wanting a slot
|
||||
VT_REQUIRE(A.deliver_one()); // A's target dropped to 0 -- sheds its held slot
|
||||
VT_CHECK_EQ(A.workers, 0u);
|
||||
VT_REQUIRE(B.deliver_one()); // B's target rose to 1 -- the plain target-changed path
|
||||
VT_CHECK_EQ(B.workers, 1u); // B took the freed slot
|
||||
}
|
||||
|
||||
VT_TEST(budget_wait_list_wakes_a_task_whose_target_never_changed) {
|
||||
// The real gap. max_active=2. Y alone, holds both (target=2). X arrives wanting 1:
|
||||
// this recompute correctly drops Y's target to 1 (giving X its guaranteed slot) and
|
||||
// raises X's target to 1 -- both real target changes, both queued. Deliver X's
|
||||
// *first*: X's target says grow, but Y still physically holds 2 (hasn't processed
|
||||
// its own shrink yet) -- confirm_slot() must deny X here (active_ == max_active_),
|
||||
// which is the correctness fix (over-admission is the real RSS bug). Then Y
|
||||
// processes its shrink and actually releases. X's target never changes again -- it
|
||||
// was already correctly 1 -- so nothing in the plain mechanism ever revisits X.
|
||||
SegmentBudget b({.max_active_segments = 2});
|
||||
QueuedTask Y{&b, tid(1)}, X{&b, tid(2)};
|
||||
b.register_task(Y.id, {.host = "h", .per_task_cap = 2, .resumable = true},
|
||||
[&](std::uint32_t n) { Y.on_target(n); });
|
||||
b.set_want(Y.id, 2);
|
||||
VT_REQUIRE(Y.deliver_one());
|
||||
VT_CHECK_EQ(Y.workers, 2u);
|
||||
|
||||
b.register_task(X.id, {.host = "h", .per_task_cap = 1, .resumable = true},
|
||||
[&](std::uint32_t n) { X.on_target(n); });
|
||||
b.set_task_order(std::vector<TaskId>{Y.id, X.id});
|
||||
b.set_want(X.id, 1);
|
||||
|
||||
VT_REQUIRE(X.deliver_one());
|
||||
VT_CHECK_EQ(X.workers, 0u); // denied: active_ == max_active_, even though X's target is 1
|
||||
|
||||
VT_REQUIRE(Y.deliver_one());
|
||||
VT_CHECK_EQ(Y.workers, 1u); // Y actually releases its excess now
|
||||
|
||||
// The bug: without a wait-list, X.pending is empty here -- nothing was ever queued
|
||||
// for it, because X's target never changed again. X would wait forever despite its
|
||||
// target correctly saying it should hold a slot.
|
||||
VT_REQUIRE(X.deliver_one());
|
||||
VT_CHECK_EQ(X.workers, 1u);
|
||||
VT_CHECK_EQ(b.budget().active, 2u);
|
||||
}
|
||||
|
||||
@@ -62,6 +62,7 @@ struct Recorder {
|
||||
std::lock_guard lk(mu);
|
||||
states.push_back(to);
|
||||
};
|
||||
c.on_decision_needed = [this](const DecisionRequest &) { decision_calls.fetch_add(1); };
|
||||
c.on_finished = [this](Result<DownloadOutcome> r) {
|
||||
if (!fired.exchange(true))
|
||||
done.set_value(std::move(r));
|
||||
@@ -123,6 +124,31 @@ std::string server_sha(TestServer &srv, const std::string &mode, const std::stri
|
||||
return out.substr(open + 1, close - open - 1);
|
||||
}
|
||||
|
||||
// Small, deliberately identical extraction to server_sha's: GET /<mode>/sign/<size>?ttl=N
|
||||
// and pull the "url" field's value out of the {"url":..., "exp":...} JSON body.
|
||||
std::string sign_url(TestServer &srv, const std::string &mode, const std::string &size,
|
||||
int ttl_seconds) {
|
||||
std::string url =
|
||||
srv.url("/" + mode + "/sign/" + size + "?ttl=" + std::to_string(ttl_seconds));
|
||||
std::string cmd = "curl -s '" + url + "'";
|
||||
std::string out;
|
||||
if (FILE *f = ::popen(cmd.c_str(), "r")) {
|
||||
char buf[1024];
|
||||
while (std::fgets(buf, sizeof buf, f))
|
||||
out += buf;
|
||||
::pclose(f);
|
||||
}
|
||||
auto q = out.find("\"url\"");
|
||||
if (q == std::string::npos)
|
||||
return {};
|
||||
auto colon = out.find(':', q);
|
||||
auto open = out.find('"', colon);
|
||||
auto close = out.find('"', open + 1);
|
||||
if (open == std::string::npos || close == std::string::npos)
|
||||
return {};
|
||||
return out.substr(open + 1, close - open - 1);
|
||||
}
|
||||
|
||||
DownloadSpec spec_for(TestServer &srv, const std::string &urlpath, const std::string &save) {
|
||||
DownloadSpec s;
|
||||
s.url = srv.url(urlpath);
|
||||
@@ -326,3 +352,323 @@ VT_TEST(engine_401_then_provide_auth_completes) {
|
||||
VT_CHECK(rec.auth_calls.load() >= 1);
|
||||
VT_CHECK_EQ(file_size(td.file("au.bin")), 1u * 1024 * 1024);
|
||||
}
|
||||
|
||||
VT_TEST(engine_401_digest_then_provide_auth_completes) {
|
||||
// Same shape as engine_401_then_provide_auth_completes, but the challenge is HTTP
|
||||
// Digest (qop=auth) rather than Basic. provide_auth() doesn't know or care which --
|
||||
// http_client.cpp always asks libcurl for CURLAUTH_ANY (net::AuthScheme::any) and lets
|
||||
// curl negotiate against whatever WWW-Authenticate the server actually sent -- so this
|
||||
// exists purely to prove that's true end-to-end, not just at the unit level.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
DownloadHandle h;
|
||||
auto cbs = rec.cbs(&h, "test", "test");
|
||||
h = eng.start(spec_for(srv, "/401-digest/file/1M", td.file("dg.bin")), std::move(cbs));
|
||||
rec.arm(h);
|
||||
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK(rec.auth_calls.load() >= 1);
|
||||
VT_CHECK_EQ(file_size(td.file("dg.bin")), 1u * 1024 * 1024);
|
||||
auto got = hash_file(td.file("dg.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "401-digest", "1M"));
|
||||
}
|
||||
|
||||
// --- hostile-mode matrix: the four where a bug is silent corruption, not a visible
|
||||
// failure (docs/04 §5 "ask, never silently corrupt" / §7's failure-policy table). ---
|
||||
|
||||
VT_TEST(engine_etag_changes_asks_instead_of_splicing) {
|
||||
// A server that revalidates with a different ETag on every response fails an If-Range
|
||||
// on any retry or resume. That must surface as "ask the user" (server_file_changed),
|
||||
// never as a silent restart-from-offset-0 spliced onto bytes already on disk.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/throttled+etag-changes/file/2M", td.file("ec.bin")),
|
||||
rec.cbs());
|
||||
|
||||
// Get real progress on at least one segment before pausing, so resume's If-Range (only
|
||||
// sent once a segment has completed > 0) actually fires.
|
||||
for (int i = 0; i < 300 && h.progress().downloaded < 64u * 1024; ++i)
|
||||
std::this_thread::sleep_for(10ms);
|
||||
VT_REQUIRE(h.progress().downloaded >= 64u * 1024);
|
||||
h.pause();
|
||||
for (int i = 0; i < 200 && h.state() != EngineState::paused; ++i)
|
||||
std::this_thread::sleep_for(20ms);
|
||||
VT_REQUIRE(h.state() == EngineState::paused);
|
||||
h.resume();
|
||||
|
||||
for (int i = 0; i < 300 && rec.decision_calls.load() == 0; ++i)
|
||||
std::this_thread::sleep_for(20ms);
|
||||
VT_REQUIRE(rec.decision_calls.load() >= 1);
|
||||
VT_CHECK_EQ(h.state(), EngineState::paused);
|
||||
|
||||
h.decide(Decision::restart);
|
||||
auto r = rec.wait(90s);
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(file_size(td.file("ec.bin")), 2u * 1024 * 1024);
|
||||
auto got = hash_file(td.file("ec.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "throttled+etag-changes", "2M"));
|
||||
}
|
||||
|
||||
VT_TEST(engine_416_mid_download_asks_instead_of_exhausting_retries) {
|
||||
// 416-always 416s every ranged request, including the probe's own -- a live probe
|
||||
// correctly concludes "not resumable" and a plain-GET download never touches Range
|
||||
// (that path is the same shape as engine_non_resumable_single_segment). The failure
|
||||
// mode docs/04 means -- a server that *was* proven resumable dropping Range support
|
||||
// mid-download -- needs a worker to actually send Range against it, so force the
|
||||
// resumable, multi-segment assumption directly via probe_hint.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
|
||||
net::ProbeResult hint;
|
||||
hint.total_size = 256u * 1024;
|
||||
hint.last_modified = "Wed, 01 Jan 2025 00:00:00 GMT";
|
||||
hint.accept_ranges = true;
|
||||
hint.resumable = true;
|
||||
auto s = spec_for(srv, "/416-always/file/256K", td.file("rb.bin"));
|
||||
s.probe_hint = hint;
|
||||
s.segments = 2;
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
|
||||
for (int i = 0; i < 300 && rec.decision_calls.load() == 0; ++i)
|
||||
std::this_thread::sleep_for(20ms);
|
||||
VT_REQUIRE(rec.decision_calls.load() >= 1);
|
||||
VT_CHECK_EQ(h.state(), EngineState::paused);
|
||||
|
||||
// 416-always never recovers -- re-probing would just 416 again -- so the only sound
|
||||
// resolution is to stop, honestly, rather than retry the stale range until exhaustion.
|
||||
h.decide(Decision::abort);
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::range_not_satisfiable);
|
||||
VT_CHECK_EQ(::access(td.file("rb.bin").c_str(), F_OK), -1); // never declared complete
|
||||
}
|
||||
|
||||
VT_TEST(engine_lies_about_accept_ranges_demotes_without_asking) {
|
||||
// Ranges are always ignored (a plain 200, full body) but ETag/Last-Modified are stable
|
||||
// and honest -- unlike etag-changes, this is provably the *same* file, just a Range-
|
||||
// blind connection. docs/04 §7: demote to 1 segment and continue, automatically, no
|
||||
// user round-trip. As with 416-always, a live probe already gets this right up front
|
||||
// (proven non-resumable), so probe_hint forces the interesting mid-download case.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
|
||||
std::string want = server_sha(srv, "lies-about-accept-ranges", "512K");
|
||||
VT_REQUIRE(!want.empty());
|
||||
net::ProbeResult hint;
|
||||
hint.total_size = 512u * 1024;
|
||||
hint.last_modified = "Wed, 01 Jan 2025 00:00:00 GMT"; // testserver sends this verbatim
|
||||
hint.accept_ranges = true;
|
||||
hint.resumable = true;
|
||||
auto s = spec_for(srv, "/lies-about-accept-ranges/file/512K", td.file("lar.bin"));
|
||||
s.probe_hint = hint;
|
||||
s.segments = 4;
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
|
||||
auto r = rec.wait(60s);
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(rec.decision_calls.load(), 0); // demoted automatically, not asked
|
||||
VT_CHECK_EQ(file_size(td.file("lar.bin")), 512u * 1024);
|
||||
auto got = hash_file(td.file("lar.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), want);
|
||||
}
|
||||
|
||||
VT_TEST(engine_content_length_mismatch_fails_honestly) {
|
||||
// Content-Length promises the true size but the connection always closes short of it.
|
||||
// There is no recovery (unlike flaky-reset, this never "heals" on a later attempt), so
|
||||
// the segment's remaining range shrinks every retry until it stalls at zero progress.
|
||||
// The only correct outcome is a real, visible failure -- never a rename to save_path
|
||||
// built from a file that is quietly missing bytes.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto s = spec_for(srv, "/content-length-mismatch/file/16K", td.file("clm.bin"));
|
||||
s.segments = 1;
|
||||
s.max_retries = 4;
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
auto r = rec.wait(60s);
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::max_retries_exhausted);
|
||||
VT_CHECK(rec.saw(EngineState::failed));
|
||||
VT_CHECK_EQ(::access(td.file("clm.bin").c_str(), F_OK), -1); // never renamed into place
|
||||
}
|
||||
|
||||
// --- remaining hostile-mode matrix (tools/testserver/README.md's mode table). ---
|
||||
|
||||
VT_TEST(engine_expiring_signed_url_recovers_via_refresh_url) {
|
||||
// A signed URL past its ttl 403s (tools/testserver's own JSON body distinguishes
|
||||
// "expired" from "bad signature", but core never parses response bodies -- CLAUDE.md
|
||||
// §3 -- so both just read as a 403). The one automatic referrer retry (see
|
||||
// engine_403_without_referer_retries_with_origin, below) can't fix an expired
|
||||
// signature, so the second 403 asks -- via the same auto_pause_locked(..., false,
|
||||
// true) "ask, don't just fail" path as wrong_status/range_bad -- rather than
|
||||
// terminally failing outright, specifically so DownloadHandle::refresh_url() (its own
|
||||
// contract: works "on a live or paused task", never on a terminal one) stays usable:
|
||||
// the README pairs this mode with exactly that recovery.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
|
||||
std::string expired = sign_url(srv, "expiring-signed-url", "64K", /*ttl=*/1);
|
||||
VT_REQUIRE(!expired.empty());
|
||||
std::this_thread::sleep_for(1500ms); // let the ttl actually pass before the first request
|
||||
|
||||
DownloadSpec s;
|
||||
s.url = expired;
|
||||
s.save_path = td.file("exp.bin");
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
|
||||
for (int i = 0; i < 300 && rec.decision_calls.load() == 0; ++i)
|
||||
std::this_thread::sleep_for(20ms);
|
||||
VT_REQUIRE(rec.decision_calls.load() >= 1);
|
||||
VT_CHECK_EQ(h.state(), EngineState::paused);
|
||||
|
||||
std::string fresh = sign_url(srv, "expiring-signed-url", "64K", /*ttl=*/60);
|
||||
VT_REQUIRE(!fresh.empty());
|
||||
h.refresh_url(fresh);
|
||||
|
||||
auto r = rec.wait(60s);
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(file_size(td.file("exp.bin")), 64u * 1024);
|
||||
auto got = hash_file(td.file("exp.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "expiring-signed-url", "64K"));
|
||||
}
|
||||
|
||||
VT_TEST(engine_403_without_referer_retries_with_origin) {
|
||||
// docs/04 §7: "403 after redirect: retry once with the original referrer -- many CDNs
|
||||
// require it." No spec.referrer is set here (the common case for anything not
|
||||
// initiated from a browser page, e.g. `velox add <url>`), so the first attempt 403s;
|
||||
// the engine's own retry supplies the download URL's own origin as Referer, which
|
||||
// this mode accepts, and the download completes with no decision ever asked.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/403-without-referer/file/128K", td.file("ref.bin")),
|
||||
rec.cbs());
|
||||
auto r = rec.wait(30s);
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(rec.decision_calls.load(), 0); // recovered automatically, not asked
|
||||
VT_CHECK_EQ(file_size(td.file("ref.bin")), 128u * 1024);
|
||||
auto got = hash_file(td.file("ref.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "403-without-referer", "128K"));
|
||||
}
|
||||
|
||||
VT_TEST(engine_redirect_chain_follows_to_completion) {
|
||||
// 5 hops (tools/testserver's own --redirect-depth default) of a plain 302, query
|
||||
// string preserved across each. No CORE-side logic needed for this one -- libcurl's
|
||||
// own CURLOPT_FOLLOWLOCATION (RequestOptions::follow_redirects, already on) and
|
||||
// CURLOPT_MAXREDIRS (default 20, well over 5) do the whole thing -- this is here as
|
||||
// the end-to-end check that they're actually wired through both the probe and every
|
||||
// segment worker's own request, not just one of the two.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/redirect-chain/file/1M", td.file("rc.bin")), rec.cbs());
|
||||
auto r = rec.wait(30s);
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(file_size(td.file("rc.bin")), 1u * 1024 * 1024);
|
||||
auto got = hash_file(td.file("rc.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "redirect-chain", "1M"));
|
||||
}
|
||||
|
||||
VT_TEST(engine_slow_loris_stall_timeout_fires) {
|
||||
// Status line, headers, and body dribbled out one byte at a time for --loris-seconds,
|
||||
// then (if the dribble hasn't already been cut off) normal streaming -- a connection
|
||||
// that's technically alive (bytes ARE arriving, just far too slowly) but must not be
|
||||
// allowed to hang the task forever. http_client.cpp sets CURLOPT_LOW_SPEED_LIMIT/_TIME
|
||||
// (RequestOptions::low_speed_bytes_per_sec/low_speed_secs, hardcoded in
|
||||
// download_task.cpp to 1024 B/s for 30s) for exactly this.
|
||||
//
|
||||
// Every other test in this file uses TestServer's default 1s loris dribble to keep
|
||||
// runtime down, but 1s is far shorter than curl's 30s low_speed_time: a 1s trickle
|
||||
// followed by full-speed streaming never accumulates 30 CONSECUTIVE seconds under the
|
||||
// floor, so curl would never actually abort it -- the download would just complete
|
||||
// slightly late, which would make this test pass for the wrong reason (or not exercise
|
||||
// the stall timeout at all). Explicitly ask for a dribble that outlasts the 30s
|
||||
// threshold so the stall timeout is the thing actually observed firing, not assumed.
|
||||
TestServer srv(40.0);
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto s = spec_for(srv, "/slow-loris/file/64K", td.file("sl.bin"));
|
||||
s.segments = 1;
|
||||
s.max_retries = 1;
|
||||
auto h = eng.start(std::move(s), rec.cbs());
|
||||
auto r = rec.wait(60s); // stall timeout fires ~30s in; must resolve, not hang to 60s
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK(is_retryable(r.error().code) || r.error().code == Error::max_retries_exhausted);
|
||||
}
|
||||
|
||||
VT_TEST(engine_chunked_no_length_completes_single_segment) {
|
||||
// No Content-Length anywhere (HEAD gets none either, since it's the same handler path)
|
||||
// -- the probe can't know total_size or prove resumability, so this should take the
|
||||
// exact same "unknown size, one plain-GET segment" path as engine_non_resumable_single_
|
||||
// segment, just arriving there via a chunked body instead of a server that plainly
|
||||
// refuses Range. No core-side work needed if that demotion is already size-agnostic;
|
||||
// this is here to prove it, since every other test's server tells the probe the size
|
||||
// up front.
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/chunked-no-length/file/2M", td.file("ch.bin")),
|
||||
rec.cbs());
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(rec.decision_calls.load(), 0);
|
||||
VT_CHECK_EQ(file_size(td.file("ch.bin")), 2u * 1024 * 1024);
|
||||
auto got = hash_file(td.file("ch.bin"), Checksum::Algo::sha256);
|
||||
VT_CHECK_EQ(got.value(), server_sha(srv, "chunked-no-length", "2M"));
|
||||
}
|
||||
|
||||
// --- DAEMON-reported bug: Progress.speed_bps reads 0 for the whole life of a live
|
||||
// download while downloaded bytes visibly advance. DAEMON reads progress by polling
|
||||
// DownloadHandle::progress() (engine_port_core.hpp), not the on_progress push callback --
|
||||
// this exercises exactly that path. ---
|
||||
|
||||
VT_TEST(engine_polled_progress_reports_nonzero_speed) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
TmpDir td;
|
||||
Recorder rec;
|
||||
Engine eng;
|
||||
auto h = eng.start(spec_for(srv, "/throttled/file/4M", td.file("sp.bin")), rec.cbs());
|
||||
|
||||
// Give it real, sustained progress: the speed estimate only updates on a >=0.5s
|
||||
// sample window (seg_data), so a snapshot taken too early would legitimately read 0
|
||||
// even with the bug fixed. Poll until downloaded has clearly advanced twice over.
|
||||
std::uint64_t speed = 0;
|
||||
for (int i = 0; i < 400 && speed == 0; ++i) {
|
||||
std::this_thread::sleep_for(20ms);
|
||||
auto p = h.progress();
|
||||
if (p.downloaded >= 256u * 1024)
|
||||
speed = p.speed_bps;
|
||||
}
|
||||
VT_CHECK(speed > 0);
|
||||
|
||||
h.cancel(/*discard_partial=*/true);
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(!r.has_value());
|
||||
}
|
||||
|
||||
+21
-2
@@ -35,6 +35,10 @@ add_library(veloxd_store STATIC
|
||||
src/store/pairings.cpp
|
||||
src/store/settings.cpp
|
||||
src/store/tasks.cpp
|
||||
src/store/categories.cpp
|
||||
src/store/queues.cpp
|
||||
src/store/rules.cpp
|
||||
src/store/segments.cpp
|
||||
${_mig_hdr}
|
||||
)
|
||||
add_library(velox::daemon_store ALIAS veloxd_store)
|
||||
@@ -65,12 +69,14 @@ add_library(velox::daemon_sched ALIAS veloxd_sched)
|
||||
target_include_directories(veloxd_sched PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src)
|
||||
target_compile_features(veloxd_sched PUBLIC cxx_std_23)
|
||||
target_compile_options(veloxd_sched PRIVATE -Wall -Wextra -Wpedantic -Werror)
|
||||
target_link_libraries(veloxd_sched PUBLIC velox::proto velox::core veloxd_store nlohmann_json::nlohmann_json)
|
||||
target_link_libraries(veloxd_sched PUBLIC velox::proto velox::core veloxd_store veloxd_rpc nlohmann_json::nlohmann_json)
|
||||
|
||||
# --- veloxd_rpc — the RPC transports + dispatcher ------------------------------------
|
||||
add_library(veloxd_rpc STATIC
|
||||
src/rpc/runtime_dir.cpp
|
||||
src/rpc/systemd_activation.cpp
|
||||
src/rpc/event_loop.cpp
|
||||
src/rpc/event_hub.cpp
|
||||
src/rpc/uds_server.cpp
|
||||
src/rpc/ws_frame.cpp
|
||||
src/rpc/ws_handshake.cpp
|
||||
@@ -80,11 +86,24 @@ add_library(veloxd_rpc STATIC
|
||||
)
|
||||
add_library(velox::daemon_rpc ALIAS veloxd_rpc)
|
||||
|
||||
# Per-OS backends behind daemon/src/rpc/platform/*.hpp (docs/adr/0020, docs/08-porting.md).
|
||||
# Source selection lives in cmake/platform.cmake and nowhere else — PORT adds
|
||||
# platform/macos/ with no edit here.
|
||||
velox_platform_sources(veloxd_rpc src/rpc)
|
||||
|
||||
target_include_directories(veloxd_rpc PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src)
|
||||
target_compile_features(veloxd_rpc PUBLIC cxx_std_23)
|
||||
target_compile_options(veloxd_rpc PRIVATE -Wall -Wextra -Wpedantic -Werror)
|
||||
# velox::core: dispatcher.hpp includes sched/scheduler.hpp for the Scheduler* it drives
|
||||
# download.pause/resume/start/cancel and queue.start/stop through (D4b), which pulls in
|
||||
# core/include's vdm/*.hpp. Interface-only from here (no .cpp in this library calls into
|
||||
# CORE directly) — the actual Scheduler symbols resolve at the veloxd executable's link
|
||||
# step (veloxd links both veloxd_rpc and veloxd_sched), not here, so this does not create
|
||||
# the veloxd_rpc <-> veloxd_sched cycle that linking veloxd_sched itself would (veloxd_sched
|
||||
# already links veloxd_rpc, for EventHub).
|
||||
target_link_libraries(veloxd_rpc
|
||||
PUBLIC velox::proto veloxd_store veloxd_fs nlohmann_json::nlohmann_json Threads::Threads
|
||||
PUBLIC velox::proto velox::core veloxd_store veloxd_fs nlohmann_json::nlohmann_json
|
||||
Threads::Threads
|
||||
)
|
||||
|
||||
# --- veloxd — the daemon binary -------------------------------------------------------
|
||||
|
||||
@@ -5,9 +5,27 @@ close. Kept here (not buried in commit messages) so the next pass can see them a
|
||||
|
||||
| # | What | Where | Why deferred | Closes when |
|
||||
|---|---|---|---|---|
|
||||
| D1 | Pairing prompt is `EnvAutoApprover` (needs `VELOX_PAIR_AUTO=1`) | `rpc/pairing.hpp`, `main.cpp` | A GUI dialog / `org.freedesktop.Notifications` approver is integration work | Build step 7 (systemd + notifications) |
|
||||
| D2 | `download.probe` → `-32603` | `rpc/dispatcher.cpp` | `download.add` is wired (`fs/safepath` + store, real `-32011`); `download.probe` needs the engine's probe path for `-32013` | probe with the engine link (CORE stage 3 is landed; wire `Engine::probe`) |
|
||||
| D3 | Stub handlers for everything except `session.*`, `download.add/list/get` | `rpc/dispatcher.cpp` | No store behind them yet (categories/queues/rules/settings/limiter/schedule) | Per method, as the store query modules land behind them |
|
||||
| ~~D7~~ | **Closed — `capture.offer` is real.** Applies `capture.enabled`/`excludedHosts`/`monitoredExtensions`/`monitoredMimeTypes`/`minSizeBytes` from settings, then the rules table (`store::Rules` + CORE's `vdm::rules::match_rules`/`glob_match` — DAEMON only converts its own stored `proto::Rule` JSON into CORE's plain `vdm::rules::Rule` vocabulary, per that header's own layering note), resolves the category folder (a rule's explicit `categoryId`/`saveDir`, else `store::Categories::guess_by_extension` — the same extension-guess `download.probe`'s `suggestedCategoryId` already used, now shared instead of duplicated), dedupes against active (non-terminal) tasks by exact URL, and on `take` calls `add_one()` — the same path `download.add` itself uses — so a captured download is a real, admitted, persisted task, not a special case. The 750 ms deadline (CLAUDE.md §4 / AGENT-DAEMON.md build step 6) is checked cooperatively between every step via a new `rpc::CaptureDataSource` seam (real impl wraps `store::*`; a test fake can jump its own clock forward to simulate "the store was slow just now" with zero real sleep) — catches the realistic failure mode (several slow steps adding up) though it can't preempt one pathologically stuck single call. Verified against real `veloxd` + `tools/testserver`: a monitored-type offer answers in ~5ms and actually creates + downloads the task; an unmonitored type, an excluded host, a rule-vetoed host, and a second offer for a still-active URL all answer `ignore` with the right `reason`; a bad category save dir surfaces its real `-32011` rather than being swallowed. New `capture_offer_test` covers all of the above plus the deadline itself (two cases, one per "slow" checkpoint), asserting real wall-clock time barely moves even though the fake clock jumped 2 simulated seconds — proof the check reads the injected clock, not a disguised sleep. | `rpc/capture_data_source.hpp`, `rpc/dispatcher.{hpp,cpp}`, `store/rules.{hpp,cpp}`, `store/categories.{hpp,cpp}`, `store/tasks.{hpp,cpp}` | — | done |
|
||||
| ~~D8~~ | **Closed alongside D7** — `capture.getRules` returns the same settings-backed `enabled`/`monitoredExtensions`/`monitoredMimeTypes`/`minSizeBytes`/`excludedHosts`/`bypassModifier` capture.offer itself reads, so the two can never drift. `rulesVersion` is a constant `1` — there is no persisted revision counter yet (nothing writes `rules.*` outside this process's own lifetime to need one across a restart), and the extension already re-fetches on `event.settings.changed` regardless of what this number does; noted in case a real counter becomes worth adding later. | `rpc/dispatcher.cpp` | `rulesVersion` is a placeholder constant | — |
|
||||
| D1 | Pairing prompt is `EnvAutoApprover` (needs `VELOX_PAIR_AUTO=1`) | `rpc/pairing.hpp`, `main.cpp` | A GUI dialog / `org.freedesktop.Notifications` approver is integration work | the `org.freedesktop.Notifications` half of build step 7 — the systemd half closed as D11 below |
|
||||
| — | **D1, checked this pass, not attempted:** `libdbus-1-dev` (or `libsystemd-dev` for `sd-bus`) has no headers installed in this build environment — only the runtime `.so`s (`dpkg -l`/`apt-cache policy` confirm `libdbus-1-3` present, `libdbus-1-dev` not, "Candidate" available but not installed). A real notification-backed approver needs one of those linked into `veloxd`, which is a new build dependency for `daemon/CMakeLists.txt` (`find_package`/`pkg_check_modules`) and — since packaging manifests need to know about it too — arguably a decision to surface rather than something to reach for silently mid-session. `PairingApprover::approve()` is also still synchronous by shape (its own doc comment already says so: "the real notification-backed approver will run async and is not this shape") — swapping it for the async pattern this session built for `download.probe` (`rpc::TaskActionPort` + the server-layer deferred-reply special-case) is the right shape once there's a real implementation to justify the churn; reshaping the interface with nothing behind it yet would just be churn. Left `EnvAutoApprover` in place rather than build a fragile hand-rolled D-Bus wire client to avoid the missing headers — a broken pairing approver is worse than an honest stub. | `rpc/pairing.hpp` | missing dev headers + an undiscussed new dependency | once `libdbus-1-dev`/`libsystemd-dev` is available and the dependency is approved |
|
||||
| ~~D2~~ | **Closed** — `download.probe` is real on both transports. It's genuinely async (the engine's probe pool, up to the schema's 30s `x-deadlineMs`) and so cannot fit `VeloxDispatcher::on_download_probe`'s synchronous `HandlerResult<T>` return — `uds_server.cpp`/`ws_server.cpp` special-case `"download.probe"` before the generic `dispatch()`, exactly the way they already special-case `session.hello`/`session.subscribe`, and queue the reply whenever the callback fires. `rpc::TaskActionPort::probe_now` (kept in proto/std terms, no `vdm::net::*`, so `veloxd_rpc` never needs `core/include`'s vdm headers) is what both transports call; `sched::Scheduler::probe_now` is the implementation — builds a `vdm::net::ProbeRequest`, runs it on the engine's probe pool, maps a failure to `-32013 ProbeFailed` (with `data.httpStatus` when there was one), and fills `suggestedCategoryId`/`suggestedSaveDir` with a plain extension match against the categories table (not the real rules engine — that's still D3). Verified live: a real probe answers in ~5ms; a bad host maps to `-32013`; a connection issuing a 10s `slow-loris` probe does not block a second connection's `download.list` (answered in ~1ms) — confirms the async design actually keeps the loop free, not just compiles. | `rpc/task_action_port.hpp`, `rpc/{uds_server,ws_server}.{hpp,cpp}`, `sched/scheduler.{cpp,hpp}` | — | done |
|
||||
| D3 | Stub handlers for the rest: `grabber.*`, `media.*` | `rpc/dispatcher.cpp` | HLS/DASH grabber and media-variant support don't exist anywhere in this build yet — a bigger feature than a store-wiring pass | M4 territory, per AGENT-DAEMON.md |
|
||||
| ~~D3d~~ | **Closed — `rules.list`/`rules.upsert`.** New `store/rules.{hpp,cpp}`: `list()` in priority order, `apply(upsert, remove)` in one transaction (an empty `ruleId` generates one; a reprioritisation and a removal land atomically, per the schema's own "never leaves the table in a half-valid state"). Found and fixed along the way: migration `0001`'s `rules` table had no column for `Rule.name` at all — every `rules.list`/`.upsert` call failed outright ("no such column: name") the first time either ran against a real `Db`, unit tests included, since `:memory:` migrates through the same path. Migration `0004` adds it. | `store/rules.{hpp,cpp}`, `rpc/dispatcher.cpp`, `store/migrations/0004_*.sql` | — | done |
|
||||
| ~~D3e~~ | **Closed — `queue.reorder`.** New `store::Queues::reorder`: `taskIds` must be an exact permutation of the queue's current membership (compared as sorted sets) or nothing is written and `-32602` names the queue; a valid permutation rewrites every member's `queue_position` in one transaction. | `store/queues.{hpp,cpp}`, `rpc/dispatcher.cpp` | — | done |
|
||||
| ~~D3f~~ | **Closed — `schedule.get`/`schedule.set`.** Thin wrapper over the `queues.schedule` column that already existed (`Queue.schema.json`'s own field, read since D3b but never independently settable). `nextRunAt` is deliberately left unset: computing it correctly needs the same local-time, DST-aware window logic `sched/schedule_window.hpp`'s `window_open()` only has half of (is-it-open-right-now, not next-transition) — real work, called out rather than approximated. The field is optional; `nullopt` is a legal answer. | `store/queues.{hpp,cpp}`, `rpc/dispatcher.cpp` | `nextRunAt` unset (documented, not silently wrong) | `nextRunAt` is its own pass |
|
||||
| ~~D3g~~ | **Closed — `limiter.get`/`limiter.set`.** Backed by the same `downloads.speedLimitEnabled`/`downloads.speedLimitBps` settings keys D9 already wired (one bag of truth, not two) — the *new* part is actually reaching the engine: `EnginePort`/`TaskActionPort` gain `set_global_speed_limit(bps)` (`0` = unlimited, `vdm::rate::TokenBucket`'s own convention), wired to `vdm::Engine::rate_limiter().set_global_limit()`. Pushed live on every `limiter.set` *and* on `Scheduler::reload_config()` (so a limit from a previous run isn't silently unlimited again after a restart — nothing else re-derives it from settings the way `connection.*` already does). `applyToRunning` is accepted but has no lever to pull differently: a single shared global bucket has no "next task only" variant, so this always behaves as if it were `true` — documented in `EnginePort::set_global_speed_limit`'s own comment. Also found, not chased further: the schema's "`globalBps: 0` with `enabled: true` means 'stop everything'" is the *opposite* of what `TokenBucket` does with `rate_bps == 0` (unlimited) — a real discrepancy, but the schema itself says the GUI "must not offer" that combination, so nothing sends it in practice. | `sched/engine_port*.hpp`, `rpc/task_action_port.hpp`, `sched/scheduler.{cpp,hpp}`, `rpc/dispatcher.cpp` | the `globalBps:0` semantic clash noted above; `applyToRunning` has no real effect | flagged for whoever owns the schema/CORE conversation next |
|
||||
| ~~D3h~~ | **Closed — `download.update`.** "Moving `saveDir` or `filename` moves the file on disk in the same operation" (the schema's own words): resolved and root-checked exactly like `download.add`'s destination, then the `.veloxpart`/`.veloxpart.meta` pair (or the finished file, if the task is `complete`) is moved with `std::filesystem::rename`, falling back to copy+remove across filesystems (`EXDEV`) — only if the resolved location actually differs from where the task already is. `categoryId`/`queueId`(appended to the end of the new queue's run order)/`description`/`segments`(1-32)/`bufferBytes`(64 KiB-16 MiB)/`checksum` all apply through a new `store::Tasks::apply_update`. New `store::Tasks::UpdatePatch`/`apply_update` and `Tasks::set_url` (download.refreshUrl's own need, split out since neither `apply_update` nor `set_probe_result` owns the base `url` column). | `store/tasks.{hpp,cpp}`, `rpc/dispatcher.cpp` | see the shared note below (null-clearing) | done |
|
||||
| ~~D3i~~ | **Closed — `download.refreshUrl`.** Same async reasoning and the same server-layer special-case (`uds_server.cpp`/`ws_server.cpp` intercept before generic `dispatch()`, exactly like `download.probe`) — a real network round trip, same 30s `x-deadlineMs`. Re-probes the new URL, flags `contentChanged` only when size or validator are *both* known and actually differ (an unknown value on either side is never itself a mismatch — "it says so rather than silently restarting" needs a real disagreement, not an absent comparison), persists the new URL and probe result, and — if the task holds a live engine handle — swaps the URL in place via a newly-widened `EnginePort::refresh_url` (now takes headers too, matching `DownloadHandle::refresh_url`'s real signature; the seam had silently dropped them). Verified against real `veloxd` + `tools/testserver`, live-handle swap covered by a `sched_scheduler_test` case (asserts the engine got `refresh_url()`, not a fresh `start()`). | `sched/engine_port*.hpp`, `rpc/task_action_port.hpp`, `sched/scheduler.{cpp,hpp}`, `rpc/{uds_server,ws_server}.{hpp,cpp}` | — | done |
|
||||
| — | **Shared note across D3h/download.update and D9/settings.set:** the generated parser collapses "field absent" and "field explicitly `null`" to the same `std::optional::nullopt` for every `optional<T>` patch field (`DownloadUpdateParamsPatch`, `Settings`) — there is no second bit on the wire path that survives into `VeloxDispatcher`. Both schemas document "an explicit null clears the field," but neither handler can act on that distinction because the information is already gone by the time either sees the parsed struct. Not something to route around locally (would mean hand-parsing raw JSON past the generated `parse<T>` for a handful of fields) — this is a generator-level gap, PROTO's to close (e.g. `std::optional<std::optional<T>>`, or a parallel "which fields were present" bitset). Until then: `categoryId`/`queueId`/`description`/`checksum` on `download.update`, and every nullable `Settings` key, can be *set* through these RPCs but never explicitly cleared back to null. | `contracts/` (generator), affects `rpc/dispatcher.cpp` | the wire distinction the schema documents doesn't survive to the handler | PROTO's generator |
|
||||
| ~~D9~~ | **Closed — `settings.get`/`settings.set`.** The field <-> `SettingKey` <-> JSON-type mapping is four pointer-to-member tables in `dispatcher.cpp` (one per C++ field type: bool, ranged int, plain string, string array) plus five enum-typed keys handled individually (`parse_XXX` already validates those); every one of the 43 `SettingKey`s now has a real default (`store::Settings::kDefaults` grew from 14 entries to 43 — `capture.monitoredExtensions`'s default is the union of every builtin category's extensions, so the two never drift apart). `settings.get` honors `keys: null` = everything. `settings.set` validates every field *before* writing any of them (numeric min/max — the schema itself carries none of this, so it's hand-checked against each key's documented range; `-32602` names the offending key, its value, and its bounds) and validates `saveTo.*` paths against `fs::resolve_target`/`canonicalize_root` (`-32011`) — `saveTo.allowedRoots` entries are checked as roots in their own right, `saveTo.defaultDir`/`.tempDir` are checked as paths resolving *inside* the (possibly, in the same call, just-updated) root list. Reports exactly the keys whose *effective* value actually changed (a `set` to the value already in effect reports `changed: []`, not the key), publishes `event.settings.changed` with that same list, and calls `TaskActionPort::apply_settings_reload()` (-> `Scheduler::reload_config()`) when any `connection.*` key took effect, live rather than waiting for a restart. Verified against real `veloxd`: all 43 keys round-trip with sane defaults, a `keys` subset filters correctly, an out-of-range value is rejected with nothing else in the same call landing, `saveTo.defaultDir` outside every allowed root is `-32011`, setting `allowedRoots` and `defaultDir` together cross-validates against the *new* roots, and `event.settings.changed` fires over a live subscription. New `dispatcher_settings_test` covers the same ground without a socket. | `rpc/dispatcher.cpp`, `store/settings.{hpp,cpp}`, `rpc/task_action_port.hpp`, `sched/scheduler.hpp` | — | done |
|
||||
| ~~D3a~~ | **Closed** — `category.upsert`/`category.remove`: `store/categories.hpp` gains `get`/`upsert`/`remove`. `upsert` generates an id when absent (create) and always ignores the payload's `builtin` (preserved from the existing row on replace, false on create — a client can never mint or revoke it); the `saveDir` goes through the same `fs::resolve_target` canonicalize-and-root-check as `download.add` (`-32011` on failure). `remove` refuses a builtin at both layers (dispatcher pre-checks for the `-32602` error text; the store's own `DELETE ... AND builtin = 0` is defense in depth) and reassigns member tasks to `reassignTo` (default `"general"`) inside one transaction before deleting the row. Note: the `categories` table (0001) has no columns for `Category.mimeTypes`/`.sortOrder` — accepted on `upsert` but not persisted. | `store/categories.{hpp,cpp}`, `rpc/dispatcher.cpp` | — | done, `mimeTypes`/`sortOrder` gap noted |
|
||||
| ~~D3b~~ | **Closed** — `queue.upsert`: `store/queues.hpp` gains `get`/`upsert` (`set_state` already existed from D4b). Same create-generates-id pattern as categories; `taskIds` in the payload is ignored (schema's own note) and a create always starts `'stopped'` while a replace keeps the queue's current run state — `queue.upsert` edits config, not run state (that's `queue.start`/`stop`). Also fixed: `on_complete` was a real column since 0001 but `Queues::list`/`get` never projected it onto `Queue.onComplete` — now they do. | `store/queues.{hpp,cpp}`, `rpc/dispatcher.cpp` | — | done |
|
||||
| ~~D3c~~ | **Closed** — `download.remove`: cancels with `discard_partial=true` through `TaskActionPort` (always drops any `.veloxpart`/`.veloxpart.meta` — the row is gone either way, unlike `download.cancel`, which keeps them), deletes the finished file only when `deleteFile` is true and the task was `complete` (best-effort — a missing file doesn't fail the call), deletes the row (segments cascade via the FK), and publishes `event.task.removed` (closing the last open note under D5). `download.addBatch`: `on_download_add`'s body is now a shared `add_one()`, called once per item after merging each item's unset fields against `params.defaults`. `download.provideAuth`: forwards to `EnginePort::provide_auth` through a new `TaskActionPort::provide_auth`; `remember`/persisting to the Secret Service is accepted but not acted on — nothing in this build talks to libsecret yet (verified: no such integration exists anywhere in the tree). Verified against real `veloxd` + `tools/testserver`: category create/replace/remove-with-reassignment, queue create/replace-keeps-state, a batch add with shared `defaults.saveDir`, and remove-with-deleteFile actually deleting the file and the task then 404ing `download.get` with `-32010`. | `rpc/dispatcher.{hpp,cpp}`, `rpc/task_action_port.hpp`, `sched/scheduler.{cpp,hpp}` | `download.provideAuth`'s `remember` (needs the Secret Service, unbuilt) | done, `remember` persistence gap noted |
|
||||
| ~~D4a~~ | **Closed** — `sched/engine_port_core.hpp` wraps `vdm::Engine` + `segment_budget()`; `main.cpp` constructs `Engine` + `Scheduler`, calls `reconcile_after_restart` / `reload_config` / `tick` at startup | — | — | done (`lane/core` stage 8 merged) |
|
||||
| D4b | timer + nudges: a 1 s `timerfd` re-runs `Scheduler::tick()` and `download.add` nudges via `on_mutation`. `download.pause`/`resume`/`start`/`cancel` and the queue.* handlers still don't touch the scheduler | `rpc/dispatcher.cpp` | those handlers are still stubs (D3) | as each handler is implemented behind the store, it calls `on_mutation` / drives the scheduler |
|
||||
| D5 | `event.*` fan-out not implemented; `session.subscribe` accepts and echoes but nothing is emitted | `rpc/uds_server.cpp`, `rpc/ws_server.cpp` | No task state to broadcast until the engine is wired. `Scheduler::on_engine_state` is the hook it will fire from | With D4a — the same engine-state callback feeds both the store and `event.task.state` |
|
||||
| ~~D4b~~ | **Closed** — `download.pause`/`resume`/`start`/`cancel` and `queue.start`/`stop` all drive the scheduler now, and apply *immediately* (not deferred to the next tick — pausing/resuming/cancelling a live transfer can't wait up to 1s, and per ADR 0013 §3 the governor never touches a user-owned pause on its own). New `rpc::TaskActionPort` interface (owned by `rpc/`, implemented by `sched::Scheduler`) is the seam dispatcher.hpp depends on instead of `sched/scheduler.hpp` directly — avoids a real `veloxd_rpc` <-> `veloxd_sched` circular library dependency (`veloxd_sched` already links `veloxd_rpc` for `EventHub`). `Scheduler::user_pause/resume/start/cancel` + `pause_queue` engine-call-then-eager-transition, matching `tick()`'s existing `to_pause` pattern. Fixed a real bug hit while building this: `transition()` always overwrote `pause_reason` to NULL when the engine's own delayed pause-ack callback arrived with no explicit reason, clobbering whatever the actual initiator (user or governor) had just written — now it preserves the stored reason when none is supplied. Verified against real `veloxd` + `tools/testserver`: pausing a live single-segment throttled transfer freezes `downloadedBytes`, resume continues it from that point, cancel stops it; `queue.stop(pauseRunning:true)` pauses the queue's running task immediately. NOTE: `download.start`'s contract "a task in 'queued' jumps its queue" (priority bump) is not implemented — admission is still plain FIFO by `created_at`. | `sched/scheduler.{cpp,hpp}`, `rpc/task_action_port.hpp`, `rpc/dispatcher.{hpp,cpp}`, `store/queues.{cpp,hpp}` | — | done, except the queue-jump priority bump noted above |
|
||||
| ~~D5~~ | **Mostly closed** — `rpc/event_hub` fans out per-subscription; `session.subscribe` on both transports registers/updates/tears down a real subscription; `Scheduler::transition()` publishes `event.task.state` (with `previousState`) on every state change, scheduler-driven or engine-reported; `dispatcher::on_download_add` publishes `event.task.added`; a 250 ms timer batches `Scheduler::progress_snapshot()` into one `event.task.progress` array per AGENT-DAEMON.md item 5 / the schema's `x-maxRateHz: 4`. Verified live end to end. | — | `event.task.removed` has no source yet (`download.remove` is D3); `event.speed.global`, `event.notify`, `event.auth.required`, `event.settings.changed`, `event.grabber.progress` are unpublished — each lands with its owning handler | as each owning D3 handler lands |
|
||||
| ~~D6~~ | **Closed** — engine numbers now reach the store: `Scheduler::tick()` probes (`EnginePort::probe`) before every `start()`, persisting `sizeBytes`/`resumable`/validators via `Tasks::set_probe_result` before a byte moves; `Scheduler::persist_progress()` (called from `progress_snapshot()` *and* once more from `on_engine_state` right before `release()`/unmap on every terminal transition) writes `downloadedBytes`/`speedBps`/`segments`/`segmentDetail` from the engine's `Progress`, so a task that finishes between two 250 ms ticks (the common case for anything small or fast) still leaves real numbers instead of the pre-persistence defaults. `TaskSummary.segments` is sourced from `segments.size()` when the task has any (matching what actually lands in `segmentDetail`, per the schema's "exactly `segments` entries"), falling back to the engine's `effective_segments` (budget slots *held*, not necessarily physical range count — see `core/include/vdm/task/download.hpp`'s `Progress` comment) only pre-segmentation. `Tasks::set_final_bytes` tops up `on_finished`'s byte count as a last-resort backstop. Migration `0002` adds `speed_bps` to both `tasks` and `segments`, and fixes `segments.state`'s CHECK to include `'pending'` (0001 omitted it, so a pre-connect snapshot could never be written). Verified against real `veloxd` + `tools/testserver` (not just unit tests): `download.list`/`download.get` correct immediately after completion and after a daemon restart. | `sched/scheduler.{cpp,hpp}`, `store/{tasks,segments}.{cpp,hpp}`, `store/migrations/0002_*.sql` | — | done |
|
||||
| ~~D11~~ | **Closed — build order items 7 (the systemd half) and 9: `velox-nmhost`, socket activation, the systemd user units, and `velox(1)`.** `nmhost/src/main.cpp` (185 lines): a `poll()`-driven byte pump between Firefox's native-messaging framing on stdio (4-byte native-byte-order length prefix) and `veloxd`'s own NDJSON framing on the Unix socket — reframes each direction, no JSON parsing, no retry/backoff, exits the moment either side closes. Deliberately dependency-free (no `veloxd_*` library, no `nlohmann_json`) since it runs unconfined outside Firefox's sandbox whatever the packaging format. Two real bugs found and fixed while getting the integration test to actually pass rather than hang: (1) never set the pumped fds non-blocking, so the "drain what's available" read loop blocked on its own second `read()` instead of returning to `poll()`; (2) stdin and stdout are two different descriptors (0 and 1), not one — an early draft polled `POLLOUT` on fd 0, which is opened read-only, so EOF/writability were never both observable through the same `pollfd` entry. Both are exactly the class of bug a "trivial pump" invites and unit tests over the real binary (not just its helper functions) exist specifically to catch. `packaging/nativehost/com.velox.host.json` + its own `README.md` supersede `AGENT-DAEMON.md`'s stale "four locations" (spike S1 / ADR 0003 found only three are real — the fourth, `~/snap/firefox/common/.mozilla/...`, is not read by snap Firefox at all) and spell out the per-user-manifest / postinst implication for PKG/QA. `EnginePort`-style: `rpc/systemd_activation.cpp` is a from-scratch `sd_listen_fds()` (env vars only, no `libsystemd` link — `LISTEN_PID`/`LISTEN_FDS`, fd 3) that `UdsServer::start()` checks first, skipping its own create/bind/chmod/listen when systemd already bound the socket; `packaging/systemd/velox.socket` + `velox.service` are the unit pair, verified both by `systemd-analyze verify` and by an actual fork/dup2/execve simulation of the activation handshake (a real `session.hello` round-tripped over the handed-off fd with no `bind()` ever called inside the daemon for that run). `velox.service` deliberately skips `ProtectSystem=`/`ProtectHome=`/`ReadWritePaths=` — `saveTo.allowedRoots` is user-configurable to anywhere on the filesystem, and a sandbox here would turn a legitimately-configured save location into an opaque `EROFS`/`EACCES` instead of the daemon's own clear `-32011`. `cli/man/velox.1` documents the CLI as it actually exists today (`add`/`ls`/`pause`/`resume`/`rm`, `--json`, the three-tier `queue`/`settings` subcommands `AGENT-DAEMON.md` build step 8 originally sketched are not implemented in `cli/src/main.cpp` yet, so the page doesn't claim they are) — checked warning-free with `groff -mandoc -ww -z`. | `nmhost/{CMakeLists.txt,src/main.cpp,tests/}`, `daemon/src/rpc/{systemd_activation.{hpp,cpp},uds_server.cpp}`, `packaging/{nativehost,systemd}/`, `cli/man/velox.1` | — | done |
|
||||
| ~~D12~~ | **Closed — ADR 0020 Phase 0 for `daemon/`.** Pure refactor, zero behaviour change: the four Linux-only surfaces docs/08-porting.md lists for this lane now sit behind seams in `daemon/src/rpc/platform/{wakeup,peer_id,instance_lock,runtime_paths}.hpp`, with today's syscalls moved unchanged into `platform/linux/*.cpp` (`eventfd` -> `platform::Wakeup`, `SO_PEERCRED`/`struct ucred` -> `platform::peer_of` — same-UID check preserved verbatim as the security property, not touched — abstract-namespace socket -> `platform::acquire_instance_lock`, XDG env lookups -> `platform::runtime_base_dir`/`data_base_dir`, with the 0700-and-owned enforcement itself staying portable POSIX logic in `rpc/runtime_dir.cpp` rather than moving). No fifth seam for `timerfd`: both daemon timers (1 s scheduler tick, 250 ms progress batch) now go through a new portable `EventLoop::add_timer`, which folds the next timer deadline into the existing `poll()` timeout instead of a platform timer fd — exactly the case docs/08-porting.md calls out as not needing a backend ("the loop already has a deadline set"), so it stays in `rpc/event_loop.{hpp,cpp}` untouched by lane PORT. `daemon/CMakeLists.txt` lists the four `platform/linux/*.cpp` files directly (no `cmake/platform.cmake` yet — that and `platform/macos/` are PORT's, per the CLAUDE.md lane table added alongside ADR 0020); PORT's Phase 1 replaces the hardcoded list with `velox_platform_sources()`. Old `rpc/single_instance.{hpp,cpp}` deleted (folded into the new seam); `single_instance_test.cpp` retargeted at `rpc/platform/instance_lock.hpp` with no behaviour change. Full suite 59/59 green; `git diff` outside `platform/linux/` is moves/call-site updates only, no `#ifdef` anywhere in `daemon/`. | `rpc/platform/{wakeup,peer_id,instance_lock,runtime_paths}.hpp`, `rpc/platform/linux/*.cpp`, `rpc/event_loop.{hpp,cpp}`, `rpc/{uds_server,runtime_dir}.cpp`, `main.cpp`, `CMakeLists.txt` | — | done |
|
||||
| — | ~~Observed, not fixed (CORE, not this lane)~~ — **routed to CORE by the user.** `vdm::task::Progress.speed_bps` reads back as `0` for the whole lifetime of a live, real (non-fake) throttled download, despite `downloadedBytes` visibly advancing between polls — `core/src/task/download_task.cpp`'s per-worker EWMA never seems to produce a nonzero aggregate in this build. DAEMON passes `EnginePort::progress()`'s `speed_bps` straight through (`Scheduler::persist_progress`); nothing in this lane drops it. Still reproduces in the D4b live checks above (0 throughout a paused/resumed/cancelled transfer whose `downloadedBytes` visibly moved) — not re-filed, since it's already CORE's. |
|
||||
|
||||
+50
-52
@@ -5,6 +5,7 @@
|
||||
// (AGENT-DAEMON.md build order, steps 1 and 3). The scheduler and the engine link land
|
||||
// next.
|
||||
|
||||
#include <chrono>
|
||||
#include <csignal>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
@@ -15,11 +16,13 @@
|
||||
#include <sys/un.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <sys/timerfd.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/event_loop.hpp"
|
||||
#include "rpc/pairing.hpp"
|
||||
#include "rpc/platform/instance_lock.hpp"
|
||||
#include "rpc/runtime_dir.hpp"
|
||||
#include "rpc/uds_server.hpp"
|
||||
#include "rpc/ws_server.hpp"
|
||||
@@ -28,6 +31,7 @@
|
||||
#include "sched/scheduler.hpp"
|
||||
#include "store/migrations.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "util/time.hpp"
|
||||
#include "vdm/engine.hpp"
|
||||
#include "version.hpp"
|
||||
|
||||
@@ -39,48 +43,25 @@ void on_signal(int) {
|
||||
if (g_loop != nullptr) g_loop->stop(); // stop() is async-signal-safe (writes an eventfd)
|
||||
}
|
||||
|
||||
// Single-instance guard: bind an abstract-namespace Unix socket whose name is unique to
|
||||
// this user. A second daemon gets EADDRINUSE and exits. The kernel reclaims an
|
||||
// abstract-namespace address when the holding process dies, so a crash never wedges it
|
||||
// (docs/01 §2). Returns the held fd (kept open for the process lifetime) or -1.
|
||||
int acquire_single_instance_lock() {
|
||||
const int fd = ::socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0);
|
||||
if (fd < 0) return -1;
|
||||
|
||||
const std::string name = std::string("velox-daemon-") + std::to_string(::geteuid());
|
||||
sockaddr_un addr{};
|
||||
addr.sun_family = AF_UNIX;
|
||||
// Leading NUL selects the abstract namespace; the name follows, not NUL-terminated.
|
||||
addr.sun_path[0] = '\0';
|
||||
std::memcpy(addr.sun_path + 1, name.c_str(), name.size());
|
||||
const socklen_t len =
|
||||
static_cast<socklen_t>(offsetof(sockaddr_un, sun_path) + 1 + name.size());
|
||||
|
||||
if (::bind(fd, reinterpret_cast<sockaddr*>(&addr), len) != 0) {
|
||||
::close(fd);
|
||||
return -1;
|
||||
}
|
||||
return fd;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
std::cout << "veloxd " << velox::daemon::kDaemonVersion << " (protocol "
|
||||
<< velox::proto::kProtocolVersion << ")\n";
|
||||
|
||||
const int lock_fd = acquire_single_instance_lock();
|
||||
if (lock_fd < 0) {
|
||||
std::cerr << "veloxd: another instance is already running for this user\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
velox::daemon::rpc::RuntimeDir rt;
|
||||
if (const auto ec = velox::daemon::rpc::resolve_runtime_dir(rt)) {
|
||||
std::cerr << "veloxd: cannot prepare runtime directory: " << ec.message() << "\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
const int lock_fd = velox::daemon::rpc::platform::acquire_instance_lock(rt.path);
|
||||
if (lock_fd < 0) {
|
||||
std::cerr << "veloxd: another instance is already running for this runtime "
|
||||
"directory (" << rt.path << ")\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
velox::daemon::rpc::EventLoop loop;
|
||||
g_loop = &loop;
|
||||
|
||||
@@ -108,10 +89,11 @@ int main() {
|
||||
}
|
||||
|
||||
// --- engine + scheduler ---------------------------------------------------------
|
||||
velox::daemon::rpc::EventHub hub;
|
||||
vdm::Engine engine;
|
||||
velox::daemon::sched::EnginePortCore engine_port(engine);
|
||||
velox::daemon::sched::Scheduler scheduler(
|
||||
*db, engine_port, velox::daemon::sched::Governor{},
|
||||
*db, engine_port, velox::daemon::sched::Governor{}, &hub,
|
||||
{/*local_now*/ {},
|
||||
/*post_to_loop*/ [&loop](std::function<void()> fn) { loop.post(std::move(fn)); }});
|
||||
|
||||
@@ -120,27 +102,47 @@ int main() {
|
||||
(void)scheduler.reload_config();
|
||||
(void)scheduler.tick(); // admit anything already queued in the DB
|
||||
|
||||
velox::daemon::rpc::VeloxDispatcher dispatcher(*db);
|
||||
velox::daemon::rpc::VeloxDispatcher dispatcher(*db, hub, &scheduler);
|
||||
dispatcher.set_on_mutation([&loop, &scheduler] {
|
||||
loop.post([&scheduler] { (void)scheduler.tick(); });
|
||||
});
|
||||
|
||||
// A 1 s timer re-runs the scheduler so schedule windows opening/closing and any
|
||||
// missed nudge are picked up. Registered on the loop, no extra thread.
|
||||
const int tick_fd = ::timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC);
|
||||
if (tick_fd >= 0) {
|
||||
itimerspec spec{};
|
||||
spec.it_value.tv_sec = 1;
|
||||
spec.it_interval.tv_sec = 1;
|
||||
::timerfd_settime(tick_fd, 0, &spec, nullptr);
|
||||
loop.add_fd(tick_fd, velox::daemon::rpc::kRead, [&](int fd, unsigned) {
|
||||
std::uint64_t ticks = 0;
|
||||
[[maybe_unused]] ssize_t n = ::read(fd, &ticks, sizeof(ticks));
|
||||
(void)scheduler.tick();
|
||||
});
|
||||
}
|
||||
// missed nudge are picked up. Expressed as a poll() timeout behind EventLoop rather
|
||||
// than a timerfd (docs/08-porting.md: "the loop already has a deadline set" — no
|
||||
// per-OS backend needed here, unlike the other three daemon/rpc seams).
|
||||
loop.add_timer(std::chrono::seconds(1), [&] { (void)scheduler.tick(); });
|
||||
|
||||
velox::daemon::rpc::UdsServer uds(loop, dispatcher, rt.socket_path());
|
||||
// event.task.progress: one array message at <=4 Hz (schema x-maxRateHz), never one
|
||||
// notification per task (AGENT-DAEMON.md item 5). 250 ms keeps every active task's
|
||||
// segment bar under 4 Hz without depending on how many tasks are running.
|
||||
loop.add_timer(std::chrono::milliseconds(250), [&] {
|
||||
const auto rows = scheduler.progress_snapshot();
|
||||
if (rows.empty()) return;
|
||||
|
||||
nlohmann::json tasks_json = nlohmann::json::array();
|
||||
for (const auto& r : rows) {
|
||||
nlohmann::json t{{"taskId", r.task_id},
|
||||
{"downloadedBytes", r.downloaded_bytes},
|
||||
{"speedBps", r.speed_bps}};
|
||||
t["etaSeconds"] = r.eta_seconds ? nlohmann::json(*r.eta_seconds) : nlohmann::json(nullptr);
|
||||
if (!r.segments.empty()) {
|
||||
nlohmann::json segs = nlohmann::json::array();
|
||||
for (const auto& s : r.segments)
|
||||
segs.push_back({{"index", s.index},
|
||||
{"downloadedBytes", s.downloaded_bytes},
|
||||
{"speedBps", s.speed_bps}});
|
||||
t["segments"] = std::move(segs);
|
||||
}
|
||||
tasks_json.push_back(std::move(t));
|
||||
}
|
||||
const nlohmann::json params{{"tasks", std::move(tasks_json)},
|
||||
{"at", velox::daemon::now_iso()}};
|
||||
hub.publish(velox::proto::Event::TaskProgress,
|
||||
velox::proto::make_notification(velox::proto::Event::TaskProgress, params));
|
||||
});
|
||||
|
||||
velox::daemon::rpc::UdsServer uds(loop, dispatcher, hub, rt.socket_path(), &scheduler);
|
||||
if (const auto ec = uds.start()) {
|
||||
std::cerr << "veloxd: cannot listen on " << rt.socket_path() << ": " << ec.message()
|
||||
<< "\n";
|
||||
@@ -154,7 +156,7 @@ int main() {
|
||||
// TODO(build step 7): replace EnvAutoApprover with a GUI-dialog / desktop-notification
|
||||
// approver. Until then pairing needs VELOX_PAIR_AUTO=1.
|
||||
velox::daemon::rpc::EnvAutoApprover approver;
|
||||
velox::daemon::rpc::WsServer ws(loop, dispatcher, *db, approver, rt);
|
||||
velox::daemon::rpc::WsServer ws(loop, dispatcher, *db, approver, hub, rt, &scheduler);
|
||||
if (const auto ec = ws.start()) {
|
||||
std::cerr << "veloxd: WebSocket transport unavailable (" << ec.message()
|
||||
<< "); the extension fallback will not work this run\n";
|
||||
@@ -165,10 +167,6 @@ int main() {
|
||||
loop.run();
|
||||
std::cout << "veloxd: shutting down\n";
|
||||
|
||||
if (tick_fd >= 0) {
|
||||
loop.del_fd(tick_fd);
|
||||
::close(tick_fd);
|
||||
}
|
||||
g_loop = nullptr;
|
||||
::close(lock_fd);
|
||||
return 0;
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
#pragma once
|
||||
|
||||
// The seam capture.offer's decision logic reads through, instead of touching store::* (or
|
||||
// the wall clock) directly — the same reasoning as rpc::TaskActionPort: it lets a test
|
||||
// substitute a fake that reports "the store took a long time just now" by advancing a
|
||||
// shared fake clock, and assert that capture.offer's deadline check actually bails to
|
||||
// `ignore` instead of pressing on, without a real sleep anywhere (deterministic, instant).
|
||||
//
|
||||
// vdm::rules::Rule (not velox::proto::Rule) on purpose: this is what
|
||||
// vdm::rules::match_rules consumes directly, so the real implementation is the only place
|
||||
// that ever converts the stored proto::Rule/JSON shape into CORE's plain vocabulary.
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "vdm/rules/match.hpp"
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
class CaptureDataSource {
|
||||
public:
|
||||
virtual ~CaptureDataSource() = default;
|
||||
|
||||
// Read at every checkpoint in the offer's decision pipeline; a fake can advance this
|
||||
// however it likes (including "jump forward 2 real seconds, instantly") to simulate a
|
||||
// slow step without ever calling sleep.
|
||||
virtual std::chrono::steady_clock::time_point now() = 0;
|
||||
|
||||
virtual bool capture_enabled() = 0;
|
||||
virtual std::vector<std::string> monitored_extensions() = 0;
|
||||
virtual std::vector<std::string> monitored_mime_types() = 0;
|
||||
virtual std::int64_t min_size_bytes() = 0;
|
||||
virtual std::vector<std::string> excluded_hosts() = 0;
|
||||
|
||||
// Enabled rules, in priority order — ready for vdm::rules::match_rules as-is.
|
||||
virtual std::vector<vdm::rules::Rule> enabled_rules() = 0;
|
||||
|
||||
// "resolve the category folder": a plain extension guess (store::Categories'
|
||||
// guess_by_extension) when no rule named a category explicitly.
|
||||
virtual std::string guess_category_id(const std::string& filename) = 0;
|
||||
virtual std::string category_save_dir(const std::string& category_id) = 0;
|
||||
virtual std::string default_save_dir() = 0;
|
||||
|
||||
// True if an active (non-terminal) task already targets this exact URL.
|
||||
virtual bool has_active_duplicate(const std::string& url) = 0;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
+1063
-56
File diff suppressed because it is too large
Load Diff
@@ -14,6 +14,9 @@
|
||||
|
||||
#include <functional>
|
||||
|
||||
#include "rpc/capture_data_source.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/task_action_port.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
@@ -21,12 +24,26 @@ namespace velox::daemon::rpc {
|
||||
|
||||
class VeloxDispatcher final : public velox::proto::Dispatcher {
|
||||
public:
|
||||
explicit VeloxDispatcher(velox::daemon::store::Db& db) : db_(db) {}
|
||||
// `actions` drives download.pause/resume/start/cancel and queue.start/stop
|
||||
// immediately (D4b) — those cannot wait for the next tick(), unlike download.add's
|
||||
// on_mutation nudge. Optional so existing tests that only exercise download.add/list/
|
||||
// get keep building with no scheduler at hand; a null actions_ makes those methods
|
||||
// answer "not implemented" instead of crashing. See rpc/task_action_port.hpp for why
|
||||
// this is an interface owned by rpc/ rather than a direct sched::Scheduler* (avoids a
|
||||
// veloxd_rpc <-> veloxd_sched circular library dependency).
|
||||
VeloxDispatcher(velox::daemon::store::Db& db, EventHub& hub, TaskActionPort* actions = nullptr)
|
||||
: db_(db), hub_(hub), actions_(actions) {}
|
||||
|
||||
// Called after a handler mutates task state (download.add for now). main.cpp wires it
|
||||
// to nudge the scheduler; unset in tests.
|
||||
void set_on_mutation(std::function<void()> fn) { on_mutation_ = std::move(fn); }
|
||||
|
||||
// Test-only seam: capture.offer normally builds its own real CaptureDataSource
|
||||
// (wrapping db_) per call. A test that needs to simulate "the store is slow right
|
||||
// now" (see rpc/capture_data_source.hpp) supplies one here instead; production code
|
||||
// never calls this.
|
||||
void set_capture_source_for_test(CaptureDataSource* src) { capture_source_for_test_ = src; }
|
||||
|
||||
velox::proto::HandlerResult<velox::proto::CaptureRules>
|
||||
on_capture_getRules(const velox::proto::CaptureGetRulesParams&) override;
|
||||
velox::proto::HandlerResult<velox::proto::CaptureOfferResult>
|
||||
@@ -106,8 +123,17 @@ public:
|
||||
on_settings_set(const velox::proto::SettingsSetParams&) override;
|
||||
|
||||
private:
|
||||
// The whole of download.add's body; on_download_add and on_download_addBatch (each
|
||||
// item merged against DownloadAddBatchParams.defaults first) both call this — exactly
|
||||
// one place turns a DownloadSpec into a stored, admitted task.
|
||||
velox::proto::HandlerResult<velox::proto::DownloadAddResult> add_one(
|
||||
const velox::proto::DownloadSpec& spec);
|
||||
|
||||
velox::daemon::store::Db& db_;
|
||||
EventHub& hub_;
|
||||
TaskActionPort* actions_;
|
||||
std::function<void()> on_mutation_;
|
||||
CaptureDataSource* capture_source_for_test_ = nullptr;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
#include "rpc/event_hub.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
namespace proto = velox::proto;
|
||||
|
||||
EventHub::SubId EventHub::subscribe(Sink sink) {
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
const SubId id = next_++;
|
||||
subs_.emplace(id, Sub{std::move(sink), {}, std::nullopt});
|
||||
return id;
|
||||
}
|
||||
|
||||
void EventHub::set_filter(SubId id, std::vector<proto::Event> events,
|
||||
std::optional<std::vector<std::string>> task_ids) {
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
if (auto it = subs_.find(id); it != subs_.end()) {
|
||||
it->second.events = std::move(events);
|
||||
it->second.task_ids = std::move(task_ids);
|
||||
}
|
||||
}
|
||||
|
||||
void EventHub::unsubscribe(SubId id) {
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
subs_.erase(id);
|
||||
}
|
||||
|
||||
void EventHub::publish(proto::Event kind, const nlohmann::json& notification,
|
||||
std::string_view task_id) {
|
||||
// Copy the sinks to call out to while holding the lock only long enough to build the
|
||||
// list — a sink runs arbitrary connection code (framing + a write syscall) and must
|
||||
// not run with mu_ held.
|
||||
std::vector<Sink> targets;
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
targets.reserve(subs_.size());
|
||||
for (const auto& [id, sub] : subs_) {
|
||||
(void)id;
|
||||
if (std::find(sub.events.begin(), sub.events.end(), kind) == sub.events.end())
|
||||
continue;
|
||||
if (!task_id.empty() && sub.task_ids &&
|
||||
std::find(sub.task_ids->begin(), sub.task_ids->end(), task_id) ==
|
||||
sub.task_ids->end())
|
||||
continue;
|
||||
targets.push_back(sub.sink);
|
||||
}
|
||||
}
|
||||
for (const auto& sink : targets) sink(notification);
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
@@ -0,0 +1,63 @@
|
||||
#pragma once
|
||||
|
||||
// Per-subscription event fan-out, shared by both transports. A connection subscribes once
|
||||
// (session.subscribe) with the event kinds and optional task-id filter it wants; publish()
|
||||
// delivers a pre-built notification to every subscription that asked for that kind and
|
||||
// passes the per-task filter.
|
||||
//
|
||||
// event.task.progress is the one call site that matters for load: it is batched by the
|
||||
// caller (Scheduler::progress_snapshot + one publish) into a single array message at
|
||||
// <=4 Hz, never one publish per task — that batching happens before this class ever sees
|
||||
// it (AGENT-DAEMON.md item 5, event.task.progress.schema.json x-maxRateHz).
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
class EventHub {
|
||||
public:
|
||||
using SubId = std::uint64_t;
|
||||
using Sink = std::function<void(const nlohmann::json&)>;
|
||||
|
||||
// Register a connection with no interest yet; session.subscribe calls set_filter to
|
||||
// actually turn events on. Returns the id to unsubscribe with on disconnect.
|
||||
SubId subscribe(Sink sink);
|
||||
|
||||
// Replaces the subscription's event set and task filter (session.subscribe replaces,
|
||||
// never adds — matches the method's own description).
|
||||
void set_filter(SubId id, std::vector<velox::proto::Event> events,
|
||||
std::optional<std::vector<std::string>> task_ids);
|
||||
|
||||
void unsubscribe(SubId id);
|
||||
|
||||
// `notification` is a complete {jsonrpc, method, params} object
|
||||
// (velox::proto::make_notification). `task_id` is matched against each subscription's
|
||||
// filter when set; empty means "not task-scoped" and reaches every subscriber of
|
||||
// `kind` regardless of their filter.
|
||||
void publish(velox::proto::Event kind, const nlohmann::json& notification,
|
||||
std::string_view task_id = {});
|
||||
|
||||
private:
|
||||
struct Sub {
|
||||
Sink sink;
|
||||
std::vector<velox::proto::Event> events;
|
||||
std::optional<std::vector<std::string>> task_ids;
|
||||
};
|
||||
|
||||
std::mutex mu_;
|
||||
std::unordered_map<SubId, Sub> subs_;
|
||||
SubId next_ = 1;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
@@ -1,9 +1,8 @@
|
||||
#include "rpc/event_loop.hpp"
|
||||
|
||||
#include <poll.h>
|
||||
#include <sys/eventfd.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cerrno>
|
||||
#include <cstdint>
|
||||
#include <stdexcept>
|
||||
@@ -12,14 +11,10 @@
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
EventLoop::EventLoop() {
|
||||
wake_fd_ = ::eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
|
||||
if (wake_fd_ < 0) throw std::runtime_error("eventfd() failed");
|
||||
fds_.emplace(wake_fd_, Entry{kRead, [this](int, unsigned) { drain_wakeup(); }});
|
||||
fds_.emplace(wakeup_.pollfd(), Entry{kRead, [this](int, unsigned) { wakeup_.drain(); }});
|
||||
}
|
||||
|
||||
EventLoop::~EventLoop() {
|
||||
if (wake_fd_ >= 0) ::close(wake_fd_);
|
||||
}
|
||||
EventLoop::~EventLoop() = default;
|
||||
|
||||
void EventLoop::add_fd(int fd, unsigned interest, Callback cb) {
|
||||
fds_[fd] = Entry{interest, std::move(cb)};
|
||||
@@ -30,15 +25,11 @@ void EventLoop::mod_fd(int fd, unsigned interest) {
|
||||
}
|
||||
|
||||
void EventLoop::del_fd(int fd) {
|
||||
if (fd == wake_fd_) return; // internal, never removed
|
||||
if (fd == wakeup_.pollfd()) return; // internal, never removed
|
||||
fds_.erase(fd);
|
||||
}
|
||||
|
||||
void EventLoop::wake() noexcept {
|
||||
const std::uint64_t one = 1;
|
||||
// Best-effort: an EAGAIN here means a wakeup is already pending, which is fine.
|
||||
[[maybe_unused]] ssize_t n = ::write(wake_fd_, &one, sizeof(one));
|
||||
}
|
||||
void EventLoop::wake() noexcept { wakeup_.signal(); }
|
||||
|
||||
void EventLoop::stop() noexcept {
|
||||
stop_requested_ = true;
|
||||
@@ -62,9 +53,39 @@ void EventLoop::drain_posts() {
|
||||
for (auto& fn : batch) fn();
|
||||
}
|
||||
|
||||
void EventLoop::drain_wakeup() noexcept {
|
||||
std::uint64_t sink = 0;
|
||||
while (::read(wake_fd_, &sink, sizeof(sink)) > 0) {
|
||||
EventLoop::TimerId EventLoop::add_timer(std::chrono::milliseconds interval,
|
||||
std::function<void()> cb) {
|
||||
const TimerId id = next_timer_id_++;
|
||||
timers_.emplace(id, Timer{std::chrono::steady_clock::now() + interval, interval,
|
||||
std::move(cb)});
|
||||
wake(); // the loop may already be blocked on a longer timeout
|
||||
return id;
|
||||
}
|
||||
|
||||
void EventLoop::remove_timer(TimerId id) { timers_.erase(id); }
|
||||
|
||||
int EventLoop::next_timeout_ms() const {
|
||||
if (timers_.empty()) return -1;
|
||||
auto soonest = timers_.begin()->second.next;
|
||||
for (const auto& [id, t] : timers_) soonest = std::min(soonest, t.next);
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
const auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(soonest - now);
|
||||
return ms.count() > 0 ? static_cast<int>(ms.count()) : 0;
|
||||
}
|
||||
|
||||
void EventLoop::run_due_timers() {
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
// Snapshot ids first: a callback may add/remove timers, which would invalidate
|
||||
// iteration over timers_ directly.
|
||||
std::vector<TimerId> due;
|
||||
for (auto& [id, t] : timers_) {
|
||||
if (t.next <= now) due.push_back(id);
|
||||
}
|
||||
for (const TimerId id : due) {
|
||||
const auto it = timers_.find(id);
|
||||
if (it == timers_.end()) continue; // removed by an earlier callback this pass
|
||||
it->second.next = now + it->second.interval;
|
||||
it->second.cb();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,18 +104,22 @@ void EventLoop::run() {
|
||||
short ev = 0;
|
||||
if (e.interest & kRead) ev |= POLLIN;
|
||||
if (e.interest & kWrite) ev |= POLLOUT;
|
||||
if (ev == 0 && fd != wake_fd_) continue;
|
||||
if (ev == 0 && fd != wakeup_.pollfd()) continue;
|
||||
pollfd p{};
|
||||
p.fd = fd;
|
||||
p.events = ev;
|
||||
pfds.push_back(p);
|
||||
}
|
||||
|
||||
const int rc = ::poll(pfds.data(), pfds.size(), -1);
|
||||
const int rc = ::poll(pfds.data(), pfds.size(), next_timeout_ms());
|
||||
if (rc < 0) {
|
||||
if (errno == EINTR) continue;
|
||||
throw std::runtime_error("poll() failed");
|
||||
}
|
||||
|
||||
drain_posts();
|
||||
run_due_timers();
|
||||
|
||||
if (rc == 0) continue;
|
||||
|
||||
// Snapshot the fds that fired before invoking any callback: a callback may erase
|
||||
@@ -104,8 +129,6 @@ void EventLoop::run() {
|
||||
if (p.revents != 0) fired.push_back(p.fd);
|
||||
}
|
||||
|
||||
drain_posts();
|
||||
|
||||
for (const int fd : fired) {
|
||||
const auto it = fds_.find(fd);
|
||||
if (it == fds_.end()) continue; // removed by an earlier callback this pass
|
||||
|
||||
@@ -10,12 +10,15 @@
|
||||
// from any thread or a signal handler — they only write() a byte to an internal eventfd.
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "rpc/platform/wakeup.hpp"
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
enum Interest : unsigned {
|
||||
@@ -58,22 +61,43 @@ public:
|
||||
// marshal an engine-thread callback back onto the RPC loop.
|
||||
void post(std::function<void()> fn);
|
||||
|
||||
using TimerId = std::uint64_t;
|
||||
|
||||
// Fire `cb` roughly every `interval` for as long as the loop runs, expressed as a
|
||||
// poll(2) timeout rather than a platform timer fd (docs/08-porting.md: "the loop
|
||||
// already has a deadline set" — no per-OS backend needed for this one). Not
|
||||
// reentrant-safe to call from inside a timer callback other than the one running.
|
||||
TimerId add_timer(std::chrono::milliseconds interval, std::function<void()> cb);
|
||||
void remove_timer(TimerId id);
|
||||
|
||||
private:
|
||||
struct Entry {
|
||||
unsigned interest;
|
||||
Callback cb;
|
||||
};
|
||||
|
||||
void drain_wakeup() noexcept;
|
||||
void drain_posts();
|
||||
struct Timer {
|
||||
std::chrono::steady_clock::time_point next;
|
||||
std::chrono::milliseconds interval;
|
||||
std::function<void()> cb;
|
||||
};
|
||||
|
||||
int wake_fd_; // eventfd, always registered
|
||||
void drain_posts();
|
||||
// Milliseconds until the next timer is due, or -1 if there are none (poll()'s "block
|
||||
// forever" convention).
|
||||
int next_timeout_ms() const;
|
||||
void run_due_timers();
|
||||
|
||||
platform::Wakeup wakeup_;
|
||||
bool running_ = false;
|
||||
std::atomic<bool> stop_requested_ = false; // set from stop(), read by run()
|
||||
std::unordered_map<int, Entry> fds_;
|
||||
|
||||
std::mutex post_mu_;
|
||||
std::vector<std::function<void()>> posts_;
|
||||
|
||||
TimerId next_timer_id_ = 1;
|
||||
std::unordered_map<TimerId, Timer> timers_;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
#pragma once
|
||||
|
||||
// Single-instance guard, keyed by the resolved runtime directory (see rpc/runtime_dir.hpp)
|
||||
// so isolated instances pointed at different runtime dirs never contend (docs/01 §2). The
|
||||
// mechanism is Linux's abstract-namespace Unix socket; macOS has no abstract namespace and
|
||||
// uses a real socket file plus flock() instead, which must unlink a stale socket left by a
|
||||
// crashed process (docs/08-porting.md "API mapping" — the abstract version got that for
|
||||
// free from the kernel).
|
||||
|
||||
#include <string>
|
||||
#include <system_error>
|
||||
|
||||
namespace velox::daemon::rpc::platform {
|
||||
|
||||
// Returns the held fd (kept open for the process lifetime; closing it releases the lock)
|
||||
// or -1 if another process already holds the lock for this exact `runtime_dir`, or on any
|
||||
// other error acquiring it.
|
||||
int acquire_instance_lock(const std::string& runtime_dir);
|
||||
|
||||
} // namespace velox::daemon::rpc::platform
|
||||
@@ -0,0 +1,38 @@
|
||||
#include "rpc/platform/instance_lock.hpp"
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstring>
|
||||
|
||||
#include <sys/socket.h>
|
||||
#include <sys/un.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "util/crypto.hpp"
|
||||
|
||||
namespace velox::daemon::rpc::platform {
|
||||
|
||||
int acquire_instance_lock(const std::string& runtime_dir) {
|
||||
const int fd = ::socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0);
|
||||
if (fd < 0) return -1;
|
||||
|
||||
// Truncated to 16 hex chars (64 bits): a collision would need two distinct runtime
|
||||
// dirs to hash together, which is not a security boundary here — the socket itself is
|
||||
// still 0600 same-UID-checked; this is only "don't let two daemons stomp each other".
|
||||
const std::string name =
|
||||
"velox-daemon-" + velox::daemon::crypto::sha256_hex(runtime_dir).substr(0, 16);
|
||||
sockaddr_un addr{};
|
||||
addr.sun_family = AF_UNIX;
|
||||
// Leading NUL selects the abstract namespace; the name follows, not NUL-terminated.
|
||||
addr.sun_path[0] = '\0';
|
||||
std::memcpy(addr.sun_path + 1, name.c_str(), name.size());
|
||||
const socklen_t len =
|
||||
static_cast<socklen_t>(offsetof(sockaddr_un, sun_path) + 1 + name.size());
|
||||
|
||||
if (::bind(fd, reinterpret_cast<sockaddr*>(&addr), len) != 0) {
|
||||
::close(fd);
|
||||
return -1;
|
||||
}
|
||||
return fd;
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::rpc::platform
|
||||
@@ -0,0 +1,17 @@
|
||||
#include "rpc/platform/peer_id.hpp"
|
||||
|
||||
#include <sys/socket.h>
|
||||
|
||||
namespace velox::daemon::rpc::platform {
|
||||
|
||||
std::error_code peer_of(int fd, PeerId& out) {
|
||||
ucred cred{};
|
||||
socklen_t len = sizeof(cred);
|
||||
if (::getsockopt(fd, SOL_SOCKET, SO_PEERCRED, &cred, &len) != 0) {
|
||||
return std::error_code(errno, std::generic_category());
|
||||
}
|
||||
out.uid = cred.uid;
|
||||
return {};
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::rpc::platform
|
||||
@@ -0,0 +1,44 @@
|
||||
#include "rpc/platform/runtime_paths.hpp"
|
||||
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cerrno>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace velox::daemon::rpc::platform {
|
||||
|
||||
namespace {
|
||||
std::error_code errc(int e) { return std::error_code(e, std::generic_category()); }
|
||||
} // namespace
|
||||
|
||||
std::error_code runtime_base_dir(std::string& out) {
|
||||
if (const char* xdg = ::getenv("XDG_RUNTIME_DIR"); xdg != nullptr && xdg[0] != '\0') {
|
||||
out = xdg;
|
||||
return {};
|
||||
}
|
||||
const std::string base = "/run/user/" + std::to_string(::geteuid());
|
||||
struct stat st{};
|
||||
if (::stat(base.c_str(), &st) != 0 || !S_ISDIR(st.st_mode)) {
|
||||
// No XDG_RUNTIME_DIR and no /run/user/<uid>: refuse rather than pick an insecure
|
||||
// fallback. The caller surfaces this as "cannot start".
|
||||
return errc(ENOENT);
|
||||
}
|
||||
out = base;
|
||||
return {};
|
||||
}
|
||||
|
||||
std::error_code data_base_dir(std::string& out) {
|
||||
if (const char* xdg = ::getenv("XDG_DATA_HOME"); xdg != nullptr && xdg[0] != '\0') {
|
||||
out = xdg;
|
||||
return {};
|
||||
}
|
||||
if (const char* home = ::getenv("HOME"); home != nullptr && home[0] != '\0') {
|
||||
out = std::string(home) + "/.local/share";
|
||||
return {};
|
||||
}
|
||||
return errc(ENOENT);
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::rpc::platform
|
||||
@@ -0,0 +1,32 @@
|
||||
#include "rpc/platform/wakeup.hpp"
|
||||
|
||||
#include <sys/eventfd.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace velox::daemon::rpc::platform {
|
||||
|
||||
Wakeup::Wakeup() {
|
||||
fd_ = ::eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
|
||||
if (fd_ < 0) throw std::runtime_error("eventfd() failed");
|
||||
}
|
||||
|
||||
Wakeup::~Wakeup() {
|
||||
if (fd_ >= 0) ::close(fd_);
|
||||
}
|
||||
|
||||
void Wakeup::signal() noexcept {
|
||||
const std::uint64_t one = 1;
|
||||
// Best-effort: an EAGAIN here means a wakeup is already pending, which is fine.
|
||||
[[maybe_unused]] ssize_t n = ::write(fd_, &one, sizeof(one));
|
||||
}
|
||||
|
||||
void Wakeup::drain() noexcept {
|
||||
std::uint64_t sink = 0;
|
||||
while (::read(fd_, &sink, sizeof(sink)) > 0) {
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::rpc::platform
|
||||
@@ -0,0 +1,22 @@
|
||||
#pragma once
|
||||
|
||||
// Identifies the process on the other end of a connected Unix-domain socket, for the
|
||||
// same-UID check that is the Unix transport's authorization boundary (docs/01 §2,
|
||||
// CLAUDE.md §4). `SO_PEERCRED`/`struct ucred` is Linux-only; macOS has `getpeereid`,
|
||||
// Windows named pipes carry a token instead (docs/08-porting.md "The seams" /
|
||||
// "API mapping"). The same-UID check itself is the security property and must not change
|
||||
// per-OS (docs/adr/0020 decision 2).
|
||||
|
||||
#include <system_error>
|
||||
|
||||
namespace velox::daemon::rpc::platform {
|
||||
|
||||
struct PeerId {
|
||||
unsigned int uid = 0;
|
||||
};
|
||||
|
||||
// On success, fills `out` with the peer's identity of the already-connected `fd`. On
|
||||
// failure, `out` is untouched and the error_code explains why (matches errno on Linux).
|
||||
std::error_code peer_of(int fd, PeerId& out);
|
||||
|
||||
} // namespace velox::daemon::rpc::platform
|
||||
@@ -0,0 +1,22 @@
|
||||
#pragma once
|
||||
|
||||
// Where the OS wants ephemeral runtime state and persistent user data to live, before
|
||||
// velox appends its own "/velox" subdirectory and applies the shared 0700-and-owned check
|
||||
// (rpc/runtime_dir.cpp — that enforcement is portable POSIX logic and stays there; only
|
||||
// "which base directory" is per-OS). Linux: XDG. macOS: $TMPDIR (runtime) and
|
||||
// ~/Library/Application Support (data) — see docs/08-porting.md "API mapping".
|
||||
|
||||
#include <string>
|
||||
#include <system_error>
|
||||
|
||||
namespace velox::daemon::rpc::platform {
|
||||
|
||||
// The base directory ephemeral runtime state (sockets, lock files) should live under,
|
||||
// e.g. "$XDG_RUNTIME_DIR" or "/run/user/<uid>" on Linux. No trailing slash.
|
||||
std::error_code runtime_base_dir(std::string& out);
|
||||
|
||||
// The base directory persistent user data should live under, e.g. "$XDG_DATA_HOME" or
|
||||
// "~/.local/share" on Linux. No trailing slash.
|
||||
std::error_code data_base_dir(std::string& out);
|
||||
|
||||
} // namespace velox::daemon::rpc::platform
|
||||
@@ -0,0 +1,36 @@
|
||||
#pragma once
|
||||
|
||||
// The event-loop wakeup primitive (docs/adr/0020, docs/08-porting.md). EventLoop uses this
|
||||
// to interrupt a blocked poll() from another thread or a signal handler — the eventfd
|
||||
// mechanics themselves are Linux-only; every other OS backend just needs something
|
||||
// poll()-able that signal()/drain() can drive the same way (docs/08 §"The seams": a
|
||||
// self-pipe on macOS, an event object on Windows).
|
||||
//
|
||||
// One implementation file per OS under platform/<os>/wakeup.cpp; this header carries no
|
||||
// OS types and no #ifdef (ADR 0020 decision 1).
|
||||
|
||||
namespace velox::daemon::rpc::platform {
|
||||
|
||||
class Wakeup {
|
||||
public:
|
||||
Wakeup();
|
||||
~Wakeup();
|
||||
|
||||
Wakeup(const Wakeup&) = delete;
|
||||
Wakeup& operator=(const Wakeup&) = delete;
|
||||
|
||||
// The fd to register with poll(2) for readability.
|
||||
int pollfd() const noexcept { return fd_; }
|
||||
|
||||
// Make pollfd() readable. Async-signal-safe and thread-safe.
|
||||
void signal() noexcept;
|
||||
|
||||
// Drain whatever signal() queued so pollfd() stops being readable. Call this from the
|
||||
// loop thread once pollfd() fires.
|
||||
void drain() noexcept;
|
||||
|
||||
private:
|
||||
int fd_ = -1;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::rpc::platform
|
||||
@@ -5,9 +5,10 @@
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cerrno>
|
||||
#include <cstdlib>
|
||||
#include <string>
|
||||
|
||||
#include "rpc/platform/runtime_paths.hpp"
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
namespace {
|
||||
@@ -33,17 +34,7 @@ std::error_code ensure_private_dir(const std::string& dir) {
|
||||
|
||||
std::error_code resolve_runtime_dir(RuntimeDir& out) {
|
||||
std::string base;
|
||||
if (const char* xdg = ::getenv("XDG_RUNTIME_DIR"); xdg != nullptr && xdg[0] != '\0') {
|
||||
base = xdg;
|
||||
} else {
|
||||
base = "/run/user/" + std::to_string(::geteuid());
|
||||
struct stat st{};
|
||||
if (::stat(base.c_str(), &st) != 0 || !S_ISDIR(st.st_mode)) {
|
||||
// No XDG_RUNTIME_DIR and no /run/user/<uid>: we refuse rather than pick an
|
||||
// insecure fallback. The caller surfaces this as "cannot start".
|
||||
return errc(ENOENT);
|
||||
}
|
||||
}
|
||||
if (auto ec = platform::runtime_base_dir(base)) return ec;
|
||||
if (!base.empty() && base.back() == '/') base.pop_back();
|
||||
|
||||
const std::string dir = base + "/velox";
|
||||
@@ -55,13 +46,7 @@ std::error_code resolve_runtime_dir(RuntimeDir& out) {
|
||||
|
||||
std::error_code resolve_data_dir(std::string& out) {
|
||||
std::string base;
|
||||
if (const char* xdg = ::getenv("XDG_DATA_HOME"); xdg != nullptr && xdg[0] != '\0') {
|
||||
base = xdg;
|
||||
} else if (const char* home = ::getenv("HOME"); home != nullptr && home[0] != '\0') {
|
||||
base = std::string(home) + "/.local/share";
|
||||
} else {
|
||||
return errc(ENOENT);
|
||||
}
|
||||
if (auto ec = platform::data_base_dir(base)) return ec;
|
||||
if (!base.empty() && base.back() == '/') base.pop_back();
|
||||
|
||||
// Create the XDG base components leniently, then the velox dir with a strict check.
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
#include "rpc/systemd_activation.hpp"
|
||||
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cstdlib>
|
||||
#include <string>
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
namespace {
|
||||
constexpr int kListenFdsStart = 3; // SD_LISTEN_FDS_START
|
||||
} // namespace
|
||||
|
||||
int systemd_activated_fd() {
|
||||
const char* pid_env = std::getenv("LISTEN_PID");
|
||||
const char* fds_env = std::getenv("LISTEN_FDS");
|
||||
int fd = -1;
|
||||
|
||||
if (pid_env != nullptr && fds_env != nullptr) {
|
||||
try {
|
||||
if (std::stol(pid_env) == static_cast<long>(::getpid()) && std::stol(fds_env) == 1) {
|
||||
fd = kListenFdsStart;
|
||||
}
|
||||
} catch (...) {
|
||||
// Malformed env from something other than systemd; treat as not activated.
|
||||
}
|
||||
}
|
||||
|
||||
// Contract: consumed once, then cleared, so a value meant for veloxd is never
|
||||
// mistaken for one meant for a process it might itself exec later.
|
||||
::unsetenv("LISTEN_PID");
|
||||
::unsetenv("LISTEN_FDS");
|
||||
::unsetenv("LISTEN_FDNAMES");
|
||||
return fd;
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
@@ -0,0 +1,20 @@
|
||||
#pragma once
|
||||
|
||||
// Minimal sd_listen_fds(3) reimplementation — one function, no libsystemd dependency, for
|
||||
// the one fd velox.socket ever hands us. See velox.socket / velox.service in
|
||||
// packaging/nativehost's systemd unit pair: the socket unit binds
|
||||
// $XDG_RUNTIME_DIR/velox/velox.sock itself (before veloxd ever runs, so the very first
|
||||
// connection attempt after boot is queued by the kernel rather than refused) and execs
|
||||
// veloxd with that listening fd already open at fd 3, LISTEN_FDS=1, LISTEN_PID=<our pid>.
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
// The systemd-activated listening socket fd, or -1 if this process was not socket-
|
||||
// activated (LISTEN_PID doesn't match our pid, or LISTEN_FDS is unset/not exactly 1 — more
|
||||
// than one would mean a unit file mismatch, since veloxd only ever asks for one socket).
|
||||
// Clears LISTEN_PID/LISTEN_FDS from the environment on the way out either way, per
|
||||
// sd_listen_fds's own contract, so a value meant for us is never mistaken for one meant for
|
||||
// a process veloxd might itself exec later.
|
||||
int systemd_activated_fd();
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
@@ -0,0 +1,96 @@
|
||||
#pragma once
|
||||
|
||||
// The seam between the dispatcher and the scheduler for user-initiated task/queue actions
|
||||
// (download.pause/resume/start/cancel, queue.stop's pauseRunning) — owned by rpc/ so
|
||||
// dispatcher.hpp (part of veloxd_rpc) never has to include sched/scheduler.hpp, which
|
||||
// would make veloxd_rpc depend on veloxd_sched at compile time. veloxd_sched already
|
||||
// depends on veloxd_rpc (for EventHub); the other direction too would be a real circular
|
||||
// library dependency, not just an inconvenience — anything linking veloxd_rpc alone (e.g.
|
||||
// the CLI's tests) would fail to link over symbols it never calls.
|
||||
//
|
||||
// sched::Scheduler implements this directly (it already lives in a library that depends on
|
||||
// rpc/, so adding an rpc-defined base costs nothing new); main.cpp hands the dispatcher a
|
||||
// `TaskActionPort*` pointing at the same Scheduler it constructs.
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
|
||||
class TaskActionPort {
|
||||
public:
|
||||
virtual ~TaskActionPort() = default;
|
||||
|
||||
// Mirrors sched::Scheduler::UserActionResult: whether the task was found at all,
|
||||
// whether it actually changed state (a task already in the target/a terminal state is
|
||||
// reported found=true, changed=false — BulkTaskResult's own "not an error" contract),
|
||||
// and its resulting/current state spelling either way.
|
||||
struct Result {
|
||||
bool found = false;
|
||||
bool changed = false;
|
||||
std::string state;
|
||||
};
|
||||
|
||||
virtual Result user_pause(const std::string& wire_id) = 0;
|
||||
virtual Result user_resume(const std::string& wire_id) = 0;
|
||||
virtual Result user_start(const std::string& wire_id) = 0;
|
||||
virtual Result user_cancel(const std::string& wire_id, bool discard_partial) = 0;
|
||||
|
||||
// queue.stop(pauseRunning=true): pause every currently-running task in `queue_id` now.
|
||||
// Returns the wire ids actually paused.
|
||||
virtual std::vector<std::string> pause_queue(const std::string& queue_id) = 0;
|
||||
|
||||
// download.provideAuth: answers a task auto-paused on a 401/407. false if the task
|
||||
// isn't currently holding a live engine handle (nothing waiting on credentials).
|
||||
// `remember` is accepted but not yet acted on — persisting to the Secret Service isn't
|
||||
// wired anywhere in this build yet (CLAUDE.md §4: never SQLite, never logs); this
|
||||
// always does the "this retry only" half. Noted in deferrals.md.
|
||||
virtual bool provide_auth(const std::string& wire_id, const std::string& username,
|
||||
const std::string& password, bool remember) = 0;
|
||||
|
||||
// download.probe (D2), the File Info dialog's own network round trip — no task row
|
||||
// involved. Genuinely async (the engine's probe pool; up to the schema's 30s
|
||||
// x-deadlineMs) and so cannot fit VeloxDispatcher's synchronous on_download_probe:
|
||||
// the RPC server layer (uds_server.cpp / ws_server.cpp) special-cases "download.probe"
|
||||
// before the generic dispatch(), the same way it already special-cases session.hello,
|
||||
// calls this, and queues the reply whenever `done` fires — on an engine thread, so the
|
||||
// implementation must marshal back to the loop before calling it, the same as every
|
||||
// other EnginePort callback. Kept in std::string/proto terms (not vdm::net::*) so this
|
||||
// header — included by dispatcher.hpp, part of veloxd_rpc — never needs core/include's
|
||||
// vdm headers; the vdm::net::ProbeRequest/ProbeResult conversion lives in sched/, which
|
||||
// already depends on vdm.
|
||||
virtual void probe_now(
|
||||
const velox::proto::DownloadProbeParams& params,
|
||||
std::function<void(velox::proto::HandlerResult<velox::proto::DownloadProbeResult>)>
|
||||
done) = 0;
|
||||
|
||||
// download.refreshUrl ("IDM's 'Refresh Download Address'"): same async reasoning and
|
||||
// the same server-layer special-case as probe_now — a real network round trip, up to
|
||||
// the schema's own 30s x-deadlineMs. Re-probes the new URL, compares size/validator
|
||||
// against what the task already has on record (contentChanged), persists the new URL
|
||||
// and probe result, and — if the task holds a live engine handle — swaps its URL
|
||||
// in-flight without losing progress.
|
||||
virtual void refresh_url(
|
||||
const std::string& wire_id, const std::string& url,
|
||||
const std::optional<velox::proto::Headers>& headers,
|
||||
const std::optional<std::vector<velox::proto::Cookie>>& cookies,
|
||||
std::function<void(velox::proto::HandlerResult<velox::proto::DownloadRefreshUrlResult>)>
|
||||
done) = 0;
|
||||
|
||||
// settings.set of a connection.* key: re-read connection.maxConcurrentDownloads /
|
||||
// maxActiveSegments and the per-host cap table, push the new caps to the engine and
|
||||
// the governor (Scheduler::reload_config's own doc comment names this exact trigger).
|
||||
// Named apply_settings_reload rather than reload_config to avoid colliding with
|
||||
// sched::Scheduler's own already-public reload_config() (returns DbResult<void>,
|
||||
// consumed by main.cpp and a unit test — kept as-is rather than reshaped to fit here).
|
||||
virtual void apply_settings_reload() = 0;
|
||||
|
||||
// limiter.set: pushed straight to the engine's shared global token bucket. 0 means
|
||||
// unlimited (the bucket's own convention).
|
||||
virtual void set_global_speed_limit(std::uint64_t bps) = 0;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::rpc
|
||||
@@ -12,7 +12,11 @@
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <fcntl.h>
|
||||
|
||||
#include "rpc/event_loop.hpp"
|
||||
#include "rpc/platform/peer_id.hpp"
|
||||
#include "rpc/systemd_activation.hpp"
|
||||
#include "version.hpp"
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
@@ -56,8 +60,10 @@ json rpc_error(const json& id, proto::ErrorCode code, std::string_view msg, json
|
||||
|
||||
} // namespace
|
||||
|
||||
UdsServer::UdsServer(EventLoop& loop, proto::Dispatcher& dispatcher, std::string socket_path)
|
||||
: loop_(loop), dispatcher_(dispatcher), path_(std::move(socket_path)) {}
|
||||
UdsServer::UdsServer(EventLoop& loop, proto::Dispatcher& dispatcher, EventHub& hub,
|
||||
std::string socket_path, TaskActionPort* actions)
|
||||
: loop_(loop), dispatcher_(dispatcher), hub_(hub), actions_(actions),
|
||||
path_(std::move(socket_path)) {}
|
||||
|
||||
UdsServer::~UdsServer() {
|
||||
for (auto& [fd, c] : conns_) {
|
||||
@@ -72,6 +78,20 @@ UdsServer::~UdsServer() {
|
||||
}
|
||||
|
||||
std::error_code UdsServer::start() {
|
||||
// velox.socket (systemd user unit, socket activation): the unit binds this path itself
|
||||
// before veloxd ever runs and hands the already-listening fd over at fd 3 — the first
|
||||
// connection after boot is queued by the kernel rather than refused, and there is no
|
||||
// window where a client sees ECONNREFUSED while the daemon is still starting. Skips
|
||||
// create/bind/chmod/listen entirely; the socket file's lifecycle (including removal on
|
||||
// stop) belongs to the unit, not to us, so bound_ stays false.
|
||||
if (const int activated = systemd_activated_fd(); activated >= 0) {
|
||||
::fcntl(activated, F_SETFL, O_NONBLOCK);
|
||||
::fcntl(activated, F_SETFD, FD_CLOEXEC);
|
||||
listen_fd_ = activated;
|
||||
loop_.add_fd(listen_fd_, kRead, [this](int, unsigned) { on_listener_readable(); });
|
||||
return {};
|
||||
}
|
||||
|
||||
if (path_.size() + 1 > sizeof(sockaddr_un::sun_path)) return errc(ENAMETOOLONG);
|
||||
|
||||
const int fd = ::socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0);
|
||||
@@ -121,10 +141,8 @@ void UdsServer::on_listener_readable() {
|
||||
break; // EMFILE/ENFILE: stop accepting this pass; loop retries on next readable
|
||||
}
|
||||
|
||||
ucred cred{};
|
||||
socklen_t len = sizeof(cred);
|
||||
if (::getsockopt(cfd, SOL_SOCKET, SO_PEERCRED, &cred, &len) != 0 ||
|
||||
cred.uid != ::geteuid()) {
|
||||
platform::PeerId peer{};
|
||||
if (platform::peer_of(cfd, peer) || peer.uid != ::geteuid()) {
|
||||
// Not the same user. The socket mode should already prevent this; refuse hard
|
||||
// regardless — this is the authorization on the Unix transport (docs/01 §2).
|
||||
::close(cfd);
|
||||
@@ -201,12 +219,82 @@ void UdsServer::handle_line(Conn& c, const std::string& line) {
|
||||
}
|
||||
}
|
||||
|
||||
if (method == "download.probe") {
|
||||
handle_download_probe(c, req);
|
||||
return;
|
||||
}
|
||||
if (method == "download.refreshUrl") {
|
||||
handle_download_refreshUrl(c, req);
|
||||
return;
|
||||
}
|
||||
|
||||
// Everything else: the generated router. It returns a null json for a notification
|
||||
// that needs no reply.
|
||||
json reply = proto::dispatch(dispatcher_, proto::Transport::Uds, req);
|
||||
if (!reply.is_null()) queue_reply(c, reply);
|
||||
}
|
||||
|
||||
void UdsServer::handle_download_probe(Conn& c, const json& request) {
|
||||
const json id = request.contains("id") ? request.at("id") : json(nullptr);
|
||||
const json params_json = request.contains("params") ? request.at("params") : json::object();
|
||||
|
||||
auto parsed = proto::parse<proto::DownloadProbeParams>(params_json, "params");
|
||||
if (!parsed) {
|
||||
queue_reply(c, rpc_error(id, proto::ErrorCode::InvalidParams, parsed.error().message,
|
||||
json{{"path", parsed.error().path}}));
|
||||
return;
|
||||
}
|
||||
if (!actions_) {
|
||||
queue_reply(c, rpc_error(id, proto::ErrorCode::InternalError,
|
||||
"not implemented in this build: download.probe"));
|
||||
return;
|
||||
}
|
||||
|
||||
const int fd = c.fd;
|
||||
actions_->probe_now(
|
||||
*parsed, [this, fd, id](proto::HandlerResult<proto::DownloadProbeResult> r) {
|
||||
auto it = conns_.find(fd);
|
||||
if (it == conns_.end()) return; // client gone while the probe was outstanding
|
||||
if (r) {
|
||||
queue_reply(*it->second, proto::make_result(id, *r));
|
||||
} else {
|
||||
queue_reply(*it->second,
|
||||
rpc_error(id, r.error().code, r.error().message, r.error().data));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void UdsServer::handle_download_refreshUrl(Conn& c, const json& request) {
|
||||
const json id = request.contains("id") ? request.at("id") : json(nullptr);
|
||||
const json params_json = request.contains("params") ? request.at("params") : json::object();
|
||||
|
||||
auto parsed = proto::parse<proto::DownloadRefreshUrlParams>(params_json, "params");
|
||||
if (!parsed) {
|
||||
queue_reply(c, rpc_error(id, proto::ErrorCode::InvalidParams, parsed.error().message,
|
||||
json{{"path", parsed.error().path}}));
|
||||
return;
|
||||
}
|
||||
if (!actions_) {
|
||||
queue_reply(c, rpc_error(id, proto::ErrorCode::InternalError,
|
||||
"not implemented in this build: download.refreshUrl"));
|
||||
return;
|
||||
}
|
||||
|
||||
const int fd = c.fd;
|
||||
actions_->refresh_url(
|
||||
parsed->taskId, parsed->url, parsed->headers, parsed->cookies,
|
||||
[this, fd, id](proto::HandlerResult<proto::DownloadRefreshUrlResult> r) {
|
||||
auto it = conns_.find(fd);
|
||||
if (it == conns_.end()) return; // client gone while the probe was outstanding
|
||||
if (r) {
|
||||
queue_reply(*it->second, proto::make_result(id, *r));
|
||||
} else {
|
||||
queue_reply(*it->second,
|
||||
rpc_error(id, r.error().code, r.error().message, r.error().data));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
bool UdsServer::handle_session_method(Conn& c, const std::string& method, const json& request,
|
||||
json& reply) {
|
||||
const json id = request.contains("id") ? request.at("id") : json(nullptr);
|
||||
@@ -251,11 +339,21 @@ bool UdsServer::handle_session_method(Conn& c, const std::string& method, const
|
||||
json{{"path", p.error().path}});
|
||||
return true;
|
||||
}
|
||||
// Event fan-out is not wired yet; accept the subscription and echo it back so a
|
||||
// client can already register its interest without erroring.
|
||||
if (!c.sub_id) {
|
||||
const int fd = c.fd;
|
||||
c.sub_id = hub_.subscribe([this, fd](const json& n) {
|
||||
if (const auto it = conns_.find(fd); it != conns_.end()) queue_reply(*it->second, n);
|
||||
});
|
||||
}
|
||||
std::vector<proto::Event> events;
|
||||
proto::SessionSubscribeResult r;
|
||||
r.ok = true;
|
||||
for (const auto& ev : p->events) r.events.emplace_back(proto::to_string(ev));
|
||||
for (const auto& ev : p->events) {
|
||||
const auto name = proto::to_string(ev);
|
||||
r.events.emplace_back(name);
|
||||
if (auto e = proto::event_from_string(name)) events.push_back(*e);
|
||||
}
|
||||
hub_.set_filter(*c.sub_id, std::move(events), p->taskIds);
|
||||
reply = proto::make_result(id, r);
|
||||
return true;
|
||||
}
|
||||
@@ -300,6 +398,7 @@ void UdsServer::flush(Conn& c) {
|
||||
|
||||
void UdsServer::close_conn(int fd) {
|
||||
if (const auto it = conns_.find(fd); it != conns_.end()) {
|
||||
if (it->second->sub_id) hub_.unsubscribe(*it->second->sub_id);
|
||||
loop_.del_fd(fd);
|
||||
::close(fd);
|
||||
conns_.erase(it);
|
||||
|
||||
@@ -13,13 +13,16 @@
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <optional>
|
||||
#include <system_error>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/ndjson.hpp"
|
||||
#include "rpc/task_action_port.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::rpc {
|
||||
@@ -28,7 +31,11 @@ class EventLoop;
|
||||
|
||||
class UdsServer {
|
||||
public:
|
||||
UdsServer(EventLoop& loop, velox::proto::Dispatcher& dispatcher, std::string socket_path);
|
||||
// `actions` is optional (nullptr in tests that don't need download.probe) — see
|
||||
// handle_download_probe's own comment for why this method can't go through the
|
||||
// generic dispatch() path like everything else.
|
||||
UdsServer(EventLoop& loop, velox::proto::Dispatcher& dispatcher, EventHub& hub,
|
||||
std::string socket_path, TaskActionPort* actions = nullptr);
|
||||
~UdsServer();
|
||||
|
||||
UdsServer(const UdsServer&) = delete;
|
||||
@@ -51,6 +58,7 @@ private:
|
||||
bool close_after_flush = false;
|
||||
bool hello_ok = false;
|
||||
std::string session_id;
|
||||
std::optional<EventHub::SubId> sub_id;
|
||||
};
|
||||
|
||||
void on_listener_readable();
|
||||
@@ -62,12 +70,24 @@ private:
|
||||
bool handle_session_method(Conn& c, const std::string& method, const nlohmann::json& request,
|
||||
nlohmann::json& reply);
|
||||
|
||||
// download.probe is genuinely async (up to the schema's 30s x-deadlineMs, on the
|
||||
// engine's probe pool) and so cannot fit the synchronous generic dispatch() path —
|
||||
// special-cased here exactly the way handle_session_method special-cases session.*.
|
||||
// Queues the reply itself, later, when actions_->probe_now()'s callback fires; does
|
||||
// nothing if the connection is gone by then (client disconnected mid-probe).
|
||||
void handle_download_probe(Conn& c, const nlohmann::json& request);
|
||||
// Same reasoning as handle_download_probe — a real network round trip, same 30s
|
||||
// x-deadlineMs.
|
||||
void handle_download_refreshUrl(Conn& c, const nlohmann::json& request);
|
||||
|
||||
void queue_reply(Conn& c, const nlohmann::json& reply);
|
||||
void flush(Conn& c);
|
||||
void close_conn(int fd);
|
||||
|
||||
EventLoop& loop_;
|
||||
velox::proto::Dispatcher& dispatcher_;
|
||||
EventHub& hub_;
|
||||
TaskActionPort* actions_;
|
||||
std::string path_;
|
||||
int listen_fd_ = -1;
|
||||
bool bound_ = false; // path_ is ours to unlink on destruction
|
||||
|
||||
@@ -60,11 +60,14 @@ json rpc_error(const json& id, proto::ErrorCode code, std::string_view msg, json
|
||||
} // namespace
|
||||
|
||||
WsServer::WsServer(EventLoop& loop, proto::Dispatcher& dispatcher, store::Db& db,
|
||||
PairingApprover& approver, RuntimeDir runtime)
|
||||
PairingApprover& approver, EventHub& hub, RuntimeDir runtime,
|
||||
TaskActionPort* actions)
|
||||
: loop_(loop),
|
||||
dispatcher_(dispatcher),
|
||||
db_(db),
|
||||
approver_(approver),
|
||||
hub_(hub),
|
||||
actions_(actions),
|
||||
runtime_(std::move(runtime)) {}
|
||||
|
||||
WsServer::~WsServer() {
|
||||
@@ -260,10 +263,80 @@ void WsServer::handle_rpc(Conn& c, const std::string& text) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (method == "download.probe") {
|
||||
handle_download_probe(c, req);
|
||||
return;
|
||||
}
|
||||
if (method == "download.refreshUrl") {
|
||||
handle_download_refreshUrl(c, req);
|
||||
return;
|
||||
}
|
||||
|
||||
json reply = proto::dispatch(dispatcher_, proto::Transport::Ws, req);
|
||||
if (!reply.is_null()) send_text(c, reply);
|
||||
}
|
||||
|
||||
void WsServer::handle_download_probe(Conn& c, const json& request) {
|
||||
const json id = request.contains("id") ? request.at("id") : json(nullptr);
|
||||
const json params_json = request.contains("params") ? request.at("params") : json::object();
|
||||
|
||||
auto parsed = proto::parse<proto::DownloadProbeParams>(params_json, "params");
|
||||
if (!parsed) {
|
||||
send_text(c, rpc_error(id, proto::ErrorCode::InvalidParams, parsed.error().message,
|
||||
json{{"path", parsed.error().path}}));
|
||||
return;
|
||||
}
|
||||
if (!actions_) {
|
||||
send_text(c, rpc_error(id, proto::ErrorCode::InternalError,
|
||||
"not implemented in this build: download.probe"));
|
||||
return;
|
||||
}
|
||||
|
||||
const int fd = c.fd;
|
||||
actions_->probe_now(
|
||||
*parsed, [this, fd, id](proto::HandlerResult<proto::DownloadProbeResult> r) {
|
||||
auto it = conns_.find(fd);
|
||||
if (it == conns_.end()) return; // client gone while the probe was outstanding
|
||||
if (r) {
|
||||
send_text(*it->second, proto::make_result(id, *r));
|
||||
} else {
|
||||
send_text(*it->second,
|
||||
rpc_error(id, r.error().code, r.error().message, r.error().data));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void WsServer::handle_download_refreshUrl(Conn& c, const json& request) {
|
||||
const json id = request.contains("id") ? request.at("id") : json(nullptr);
|
||||
const json params_json = request.contains("params") ? request.at("params") : json::object();
|
||||
|
||||
auto parsed = proto::parse<proto::DownloadRefreshUrlParams>(params_json, "params");
|
||||
if (!parsed) {
|
||||
send_text(c, rpc_error(id, proto::ErrorCode::InvalidParams, parsed.error().message,
|
||||
json{{"path", parsed.error().path}}));
|
||||
return;
|
||||
}
|
||||
if (!actions_) {
|
||||
send_text(c, rpc_error(id, proto::ErrorCode::InternalError,
|
||||
"not implemented in this build: download.refreshUrl"));
|
||||
return;
|
||||
}
|
||||
|
||||
const int fd = c.fd;
|
||||
actions_->refresh_url(
|
||||
parsed->taskId, parsed->url, parsed->headers, parsed->cookies,
|
||||
[this, fd, id](proto::HandlerResult<proto::DownloadRefreshUrlResult> r) {
|
||||
auto it = conns_.find(fd);
|
||||
if (it == conns_.end()) return; // client gone while the probe was outstanding
|
||||
if (r) {
|
||||
send_text(*it->second, proto::make_result(id, *r));
|
||||
} else {
|
||||
send_text(*it->second,
|
||||
rpc_error(id, r.error().code, r.error().message, r.error().data));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
bool WsServer::handle_session_ws(Conn& c, const std::string& method, const json& request,
|
||||
json& reply) {
|
||||
const json id = request.contains("id") ? request.at("id") : json(nullptr);
|
||||
@@ -364,9 +437,21 @@ bool WsServer::handle_session_ws(Conn& c, const std::string& method, const json&
|
||||
json{{"path", p.error().path}});
|
||||
return true;
|
||||
}
|
||||
if (!c.sub_id) {
|
||||
const int fd = c.fd;
|
||||
c.sub_id = hub_.subscribe([this, fd](const json& n) {
|
||||
if (const auto it = conns_.find(fd); it != conns_.end()) send_text(*it->second, n);
|
||||
});
|
||||
}
|
||||
std::vector<proto::Event> events;
|
||||
proto::SessionSubscribeResult r;
|
||||
r.ok = true;
|
||||
for (const auto& ev : p->events) r.events.emplace_back(proto::to_string(ev));
|
||||
for (const auto& ev : p->events) {
|
||||
const auto name = proto::to_string(ev);
|
||||
r.events.emplace_back(name);
|
||||
if (auto e = proto::event_from_string(name)) events.push_back(*e);
|
||||
}
|
||||
hub_.set_filter(*c.sub_id, std::move(events), p->taskIds);
|
||||
reply = proto::make_result(id, r);
|
||||
return true;
|
||||
}
|
||||
@@ -423,6 +508,7 @@ void WsServer::flush(Conn& c) {
|
||||
|
||||
void WsServer::close_conn(int fd) {
|
||||
if (const auto it = conns_.find(fd); it != conns_.end()) {
|
||||
if (it->second->sub_id) hub_.unsubscribe(*it->second->sub_id);
|
||||
loop_.del_fd(fd);
|
||||
::close(fd);
|
||||
conns_.erase(it);
|
||||
|
||||
@@ -8,14 +8,17 @@
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <system_error>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/pairing.hpp"
|
||||
#include "rpc/runtime_dir.hpp"
|
||||
#include "rpc/task_action_port.hpp"
|
||||
#include "rpc/ws_frame.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
@@ -29,8 +32,12 @@ class EventLoop;
|
||||
|
||||
class WsServer {
|
||||
public:
|
||||
// `actions` is optional (nullptr in tests that don't need download.probe) — see
|
||||
// handle_download_probe's own comment for why this method can't go through the
|
||||
// generic dispatch() path like everything else.
|
||||
WsServer(EventLoop& loop, velox::proto::Dispatcher& dispatcher, velox::daemon::store::Db& db,
|
||||
PairingApprover& approver, RuntimeDir runtime);
|
||||
PairingApprover& approver, EventHub& hub, RuntimeDir runtime,
|
||||
TaskActionPort* actions = nullptr);
|
||||
~WsServer();
|
||||
|
||||
WsServer(const WsServer&) = delete;
|
||||
@@ -60,6 +67,7 @@ private:
|
||||
bool authed = false;
|
||||
std::string pairing_id;
|
||||
std::string session_id;
|
||||
std::optional<EventHub::SubId> sub_id;
|
||||
};
|
||||
|
||||
void on_listener_readable();
|
||||
@@ -70,6 +78,11 @@ private:
|
||||
bool handle_session_ws(Conn& c, const std::string& method, const nlohmann::json& request,
|
||||
nlohmann::json& reply);
|
||||
|
||||
// See UdsServer::handle_download_probe — same reasoning, same pattern, duplicated per
|
||||
// transport because each owns its own Conn/send mechanics.
|
||||
void handle_download_probe(Conn& c, const nlohmann::json& request);
|
||||
void handle_download_refreshUrl(Conn& c, const nlohmann::json& request);
|
||||
|
||||
void send_text(Conn& c, const nlohmann::json& value);
|
||||
void send_frame(Conn& c, WsOpcode op, std::string_view payload);
|
||||
void begin_close(Conn& c, std::uint16_t code, std::string_view reason);
|
||||
@@ -80,6 +93,8 @@ private:
|
||||
velox::proto::Dispatcher& dispatcher_;
|
||||
velox::daemon::store::Db& db_;
|
||||
PairingApprover& approver_;
|
||||
EventHub& hub_;
|
||||
TaskActionPort* actions_;
|
||||
RuntimeDir runtime_;
|
||||
PairingRateLimiter rate_limiter_;
|
||||
|
||||
|
||||
@@ -7,18 +7,22 @@
|
||||
// wraps `vdm::Engine` + `vdm::segment::SegmentBudget`; FakeEnginePort records calls.
|
||||
//
|
||||
// Task ids here are `vdm::TaskId` — the engine assigns one from start() and the Scheduler
|
||||
// keeps the wire-UUID <-> TaskId map (ADR 0013). Admission is the Scheduler's: it calls
|
||||
// start() only for a task the governor admitted, and the engine begins probing at once
|
||||
// (it does not queue). The min-1 fairness rule in SegmentBudget then guarantees each
|
||||
// started task a slot; set_task_order pushes the priority.
|
||||
// keeps the wire-UUID <-> TaskId map (ADR 0013). Admission is the Scheduler's: for a task
|
||||
// the governor admits, it probes first (persisting sizeBytes/resumable/validator before a
|
||||
// single byte moves), then calls start() with that ProbeResult as probe_hint. The min-1
|
||||
// fairness rule in SegmentBudget then guarantees each started task a slot; set_task_order
|
||||
// pushes the priority.
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "vdm/ids.hpp"
|
||||
#include "vdm/net/probe.hpp"
|
||||
#include "vdm/task/download.hpp"
|
||||
#include "vdm/util/result.hpp"
|
||||
|
||||
namespace velox::daemon::sched {
|
||||
|
||||
@@ -26,6 +30,13 @@ class EnginePort {
|
||||
public:
|
||||
virtual ~EnginePort() = default;
|
||||
|
||||
// Runs on the probe pool, outside the segment budget (ADR 0011 §5); `done` arrives on
|
||||
// an engine thread, exactly once. The Scheduler probes before every start() so it
|
||||
// always has a real ProbeResult (size, resumable, validator) to persist and to pass
|
||||
// back as DownloadSpec.probe_hint — one code path instead of "sometimes has one".
|
||||
virtual void probe(const vdm::net::ProbeRequest& req,
|
||||
std::function<void(vdm::Result<vdm::net::ProbeResult>)> done) = 0;
|
||||
|
||||
virtual vdm::TaskId start(const vdm::task::DownloadSpec& spec,
|
||||
vdm::task::DownloadCallbacks callbacks) = 0;
|
||||
|
||||
@@ -35,15 +46,26 @@ public:
|
||||
virtual void provide_auth(vdm::TaskId, const std::string& username,
|
||||
const std::string& password, bool remember) = 0;
|
||||
virtual void decide(vdm::TaskId, vdm::task::Decision) = 0;
|
||||
virtual void refresh_url(vdm::TaskId, const std::string& url) = 0;
|
||||
virtual void refresh_url(vdm::TaskId, const std::string& url,
|
||||
const std::vector<vdm::net::HeaderField>& headers = {}) = 0;
|
||||
|
||||
// The daemon is done with this task (it went terminal). Drop the handle. Idempotent.
|
||||
virtual void release(vdm::TaskId) = 0;
|
||||
|
||||
// A synchronous, lock-guarded snapshot (DownloadHandle::progress()). nullopt if the
|
||||
// id is unknown (already released, or never started).
|
||||
virtual std::optional<vdm::task::Progress> progress(vdm::TaskId) const = 0;
|
||||
|
||||
// ADR 0011 admission surface. Values are DAEMON's; enforcement is the engine's.
|
||||
virtual void set_task_order(const std::vector<vdm::TaskId>& order) = 0;
|
||||
virtual void set_max_active_segments(std::uint32_t n) = 0;
|
||||
virtual void set_host_segment_cap(const std::string& host, std::uint32_t cap) = 0;
|
||||
|
||||
// limiter.set: 0 means unlimited (vdm::rate::TokenBucket's own convention), applied
|
||||
// across every active transfer immediately — there is no "next task only" variant for
|
||||
// a single shared global bucket, so `applyToRunning` on the wire has nothing to select
|
||||
// between; it is accepted for schema compliance and always behaves as if true.
|
||||
virtual void set_global_speed_limit(std::uint64_t bps) = 0;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::sched
|
||||
|
||||
@@ -16,6 +16,11 @@ class EnginePortCore final : public EnginePort {
|
||||
public:
|
||||
explicit EnginePortCore(vdm::Engine& engine) : engine_(engine) {}
|
||||
|
||||
void probe(const vdm::net::ProbeRequest& req,
|
||||
std::function<void(vdm::Result<vdm::net::ProbeResult>)> done) override {
|
||||
engine_.probe(req, std::move(done));
|
||||
}
|
||||
|
||||
vdm::TaskId start(const vdm::task::DownloadSpec& spec,
|
||||
vdm::task::DownloadCallbacks callbacks) override {
|
||||
vdm::task::DownloadHandle h = engine_.start(spec, std::move(callbacks));
|
||||
@@ -40,10 +45,16 @@ public:
|
||||
void decide(vdm::TaskId id, vdm::task::Decision d) override {
|
||||
if (auto* h = find(id)) h->decide(d);
|
||||
}
|
||||
void refresh_url(vdm::TaskId id, const std::string& url) override {
|
||||
if (auto* h = find(id)) h->refresh_url(url);
|
||||
void refresh_url(vdm::TaskId id, const std::string& url,
|
||||
const std::vector<vdm::net::HeaderField>& headers) override {
|
||||
if (auto* h = find(id)) h->refresh_url(url, headers);
|
||||
}
|
||||
void release(vdm::TaskId id) override { handles_.erase(id); }
|
||||
std::optional<vdm::task::Progress> progress(vdm::TaskId id) const override {
|
||||
auto it = handles_.find(id);
|
||||
if (it == handles_.end()) return std::nullopt;
|
||||
return it->second.progress();
|
||||
}
|
||||
|
||||
void set_task_order(const std::vector<vdm::TaskId>& order) override {
|
||||
engine_.segment_budget().set_task_order(order);
|
||||
@@ -54,6 +65,9 @@ public:
|
||||
void set_host_segment_cap(const std::string& host, std::uint32_t cap) override {
|
||||
engine_.segment_budget().set_host_segment_cap(host, cap);
|
||||
}
|
||||
void set_global_speed_limit(std::uint64_t bps) override {
|
||||
engine_.rate_limiter().set_global_limit(bps);
|
||||
}
|
||||
|
||||
private:
|
||||
vdm::task::DownloadHandle* find(vdm::TaskId id) {
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "sched/engine_port.hpp"
|
||||
@@ -29,6 +30,26 @@ public:
|
||||
std::vector<std::uint32_t> max_active_segments;
|
||||
std::vector<std::pair<std::string, std::uint32_t>> host_caps;
|
||||
|
||||
// probe(): synchronous by default (a default-constructed ProbeResult — success,
|
||||
// resumable=false, no known size) so a test that doesn't care about probe details
|
||||
// still sees start() happen within the same tick(). Set auto_probe_result to nullopt
|
||||
// to switch to manual mode: probe() then just records the request and stashes `done`
|
||||
// in pending_probes for the test to resolve explicitly, in order.
|
||||
std::optional<vdm::Result<vdm::net::ProbeResult>> auto_probe_result =
|
||||
vdm::Result<vdm::net::ProbeResult>{vdm::net::ProbeResult{}};
|
||||
std::vector<vdm::net::ProbeRequest> probe_requests;
|
||||
std::vector<std::function<void(vdm::Result<vdm::net::ProbeResult>)>> pending_probes;
|
||||
|
||||
void probe(const vdm::net::ProbeRequest& req,
|
||||
std::function<void(vdm::Result<vdm::net::ProbeResult>)> done) override {
|
||||
probe_requests.push_back(req);
|
||||
if (auto_probe_result) {
|
||||
done(*auto_probe_result);
|
||||
} else {
|
||||
pending_probes.push_back(std::move(done));
|
||||
}
|
||||
}
|
||||
|
||||
vdm::TaskId start(const vdm::task::DownloadSpec& spec,
|
||||
vdm::task::DownloadCallbacks callbacks) override {
|
||||
const vdm::TaskId id{next_++};
|
||||
@@ -40,13 +61,32 @@ public:
|
||||
void cancel(vdm::TaskId id, bool discard) override { cancelled.emplace_back(id, discard); }
|
||||
void provide_auth(vdm::TaskId, const std::string&, const std::string&, bool) override {}
|
||||
void decide(vdm::TaskId, vdm::task::Decision) override {}
|
||||
void refresh_url(vdm::TaskId, const std::string&) override {}
|
||||
void refresh_url(vdm::TaskId id, const std::string& url,
|
||||
const std::vector<vdm::net::HeaderField>& headers) override {
|
||||
refreshed_urls.emplace_back(id, url, headers);
|
||||
}
|
||||
struct RefreshCall {
|
||||
vdm::TaskId id;
|
||||
std::string url;
|
||||
std::vector<vdm::net::HeaderField> headers;
|
||||
};
|
||||
std::vector<RefreshCall> refreshed_urls;
|
||||
void release(vdm::TaskId id) override { released.push_back(id); }
|
||||
std::optional<vdm::task::Progress> progress(vdm::TaskId id) const override {
|
||||
auto it = fake_progress.find(id.value);
|
||||
return it == fake_progress.end() ? std::nullopt : std::optional(it->second);
|
||||
}
|
||||
|
||||
// Tests set this to control what progress(id) returns.
|
||||
std::unordered_map<std::uint64_t, vdm::task::Progress> fake_progress;
|
||||
void set_task_order(const std::vector<vdm::TaskId>& order) override { orders.push_back(order); }
|
||||
void set_max_active_segments(std::uint32_t n) override { max_active_segments.push_back(n); }
|
||||
void set_host_segment_cap(const std::string& h, std::uint32_t c) override {
|
||||
host_caps.emplace_back(h, c);
|
||||
}
|
||||
void set_global_speed_limit(std::uint64_t bps) override { global_speed_limits.push_back(bps); }
|
||||
|
||||
std::vector<std::uint64_t> global_speed_limits;
|
||||
|
||||
const std::vector<vdm::TaskId>& last_order() const { return orders.back(); }
|
||||
|
||||
|
||||
+449
-46
@@ -6,6 +6,8 @@
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "sched/schedule_window.hpp"
|
||||
#include "store/categories.hpp"
|
||||
#include "store/segments.hpp"
|
||||
#include "store/settings.hpp"
|
||||
#include "store/tasks.hpp"
|
||||
|
||||
@@ -100,10 +102,33 @@ std::vector<proto::TaskState> non_terminal_states() {
|
||||
proto::TaskState::Verifying};
|
||||
}
|
||||
|
||||
TaskErrorFields to_error_fields(const vdm::ErrorInfo& err) {
|
||||
TaskErrorFields ef;
|
||||
ef.code = std::string(vdm::error_name(err.code)); // matches TaskErrorCode by name (ADR 0010)
|
||||
ef.message = err.context;
|
||||
if (err.http_status != 0) ef.http_status = err.http_status;
|
||||
ef.retryable = err.retryable;
|
||||
return ef;
|
||||
}
|
||||
|
||||
std::string segment_state_name(vdm::segment::SegState s) {
|
||||
using S = vdm::segment::SegState;
|
||||
switch (s) {
|
||||
case S::idle: return "pending";
|
||||
case S::connecting: return "connecting";
|
||||
case S::downloading: return "downloading";
|
||||
case S::stalled: return "stalled";
|
||||
case S::complete: return "complete";
|
||||
case S::failed: return "failed";
|
||||
}
|
||||
return "pending";
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Scheduler::Scheduler(store::Db& db, EnginePort& engine, Governor governor, Deps deps)
|
||||
: db_(db), engine_(engine), governor_(std::move(governor)), deps_(std::move(deps)) {
|
||||
Scheduler::Scheduler(store::Db& db, EnginePort& engine, Governor governor, rpc::EventHub* hub,
|
||||
Deps deps)
|
||||
: db_(db), engine_(engine), governor_(std::move(governor)), hub_(hub), deps_(std::move(deps)) {
|
||||
if (!deps_.local_now) deps_.local_now = local_now_default;
|
||||
if (!deps_.post_to_loop) deps_.post_to_loop = [](std::function<void()> f) { f(); };
|
||||
}
|
||||
@@ -158,6 +183,16 @@ store::DbResult<void> Scheduler::reload_config() {
|
||||
governor_.set_config(cfg);
|
||||
engine_.set_max_active_segments(
|
||||
static_cast<std::uint32_t>(std::max<std::int64_t>(cfg.max_active_segments, 1)));
|
||||
|
||||
// The global speed limit persists across a restart the same as any other setting, but
|
||||
// (unlike connection.* above) nothing re-derives it into engine state on its own —
|
||||
// limiter.set is the only other place that calls set_global_speed_limit, and that only
|
||||
// fires on an explicit RPC in a running daemon. Push it here too so a limit set in a
|
||||
// previous run is not silently unlimited again after a restart.
|
||||
const bool limit_enabled = settings.get_bool("downloads.speedLimitEnabled");
|
||||
const std::int64_t limit_bps = settings.get_int("downloads.speedLimitBps");
|
||||
engine_.set_global_speed_limit(limit_enabled ? static_cast<std::uint64_t>(std::max<std::int64_t>(limit_bps, 0))
|
||||
: 0);
|
||||
return {};
|
||||
}
|
||||
|
||||
@@ -211,56 +246,32 @@ store::DbResult<void> Scheduler::tick() {
|
||||
const Decision d = governor_.evaluate(views, queues);
|
||||
|
||||
// --- apply ------------------------------------------------------------------
|
||||
// to_start: probe first, always — a real ProbeResult (size, resumable, validator) is
|
||||
// what makes sizeBytes/resumable correct on the wire, not a post-hoc guess. The task
|
||||
// moves to `probing` immediately so the governor does not re-admit it on the next
|
||||
// tick while the (possibly slow, always async) probe is outstanding.
|
||||
for (const auto& wire_id : d.to_start) {
|
||||
auto got = tasks.get(wire_id);
|
||||
if (!got || !got->has_value()) continue;
|
||||
const store::TaskRow& row = **got;
|
||||
|
||||
vdm::task::DownloadSpec spec;
|
||||
spec.url = row.url;
|
||||
spec.save_path = row.save_dir + "/" + row.filename;
|
||||
if (row.req_segments) spec.segments = static_cast<std::uint32_t>(*row.req_segments);
|
||||
if (row.req_buffer_bytes)
|
||||
spec.buffer_bytes = static_cast<std::uint64_t>(*row.req_buffer_bytes);
|
||||
if (row.checksum_algo && row.checksum_value) {
|
||||
vdm::task::Checksum ck;
|
||||
ck.hex = *row.checksum_value;
|
||||
if (*row.checksum_algo == "md5") ck.algo = vdm::task::Checksum::Algo::md5;
|
||||
else if (*row.checksum_algo == "sha1") ck.algo = vdm::task::Checksum::Algo::sha1;
|
||||
else if (*row.checksum_algo == "sha512") ck.algo = vdm::task::Checksum::Algo::sha512;
|
||||
else ck.algo = vdm::task::Checksum::Algo::sha256;
|
||||
spec.checksum = ck;
|
||||
}
|
||||
spec.allow_resume = true; // resume from a sidecar if one is beside save_path
|
||||
transition(wire_id, "probing", std::nullopt, std::nullopt);
|
||||
|
||||
vdm::net::ProbeRequest req;
|
||||
req.url = row.url;
|
||||
// headers / cookies / referrer / user_agent are not persisted yet (a URL-only
|
||||
// `velox add` has none); the capture path will fill them when it lands.
|
||||
|
||||
vdm::task::DownloadCallbacks cbs;
|
||||
const std::string id_copy = wire_id;
|
||||
cbs.on_state = [this, id_copy](vdm::task::EngineState, vdm::task::EngineState to,
|
||||
const std::optional<vdm::ErrorInfo>& err) {
|
||||
std::optional<TaskErrorFields> ef;
|
||||
if (err) {
|
||||
ef = TaskErrorFields{};
|
||||
ef->code = std::string(vdm::error_name(err->code)); // matches TaskErrorCode
|
||||
ef->message = err->context;
|
||||
if (err->http_status != 0) ef->http_status = err->http_status;
|
||||
ef->retryable = err->retryable;
|
||||
}
|
||||
const std::string to_name = engine_state_name(to);
|
||||
deps_.post_to_loop(
|
||||
[this, id_copy, to_name, ef]() { on_engine_state(id_copy, to_name, ef); });
|
||||
};
|
||||
|
||||
const vdm::TaskId engine_id = engine_.start(spec, std::move(cbs));
|
||||
map(wire_id, engine_id);
|
||||
(void)tasks.set_state(wire_id, "probing", std::nullopt);
|
||||
engine_.probe(req, [this, id_copy](vdm::Result<vdm::net::ProbeResult> pr) {
|
||||
deps_.post_to_loop([this, id_copy, pr]() { on_probe_result(id_copy, pr); });
|
||||
});
|
||||
}
|
||||
|
||||
for (const auto& wire_id : d.to_resume) {
|
||||
if (auto eid = engine_id_of(wire_id)) {
|
||||
engine_.resume(*eid);
|
||||
(void)tasks.set_state(wire_id, "connecting", std::nullopt);
|
||||
transition(wire_id, "connecting", std::nullopt, std::nullopt);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,7 +280,7 @@ store::DbResult<void> Scheduler::tick() {
|
||||
? pause_reason_str(d.pause_reasons.at(wire_id))
|
||||
: "user";
|
||||
if (auto eid = engine_id_of(wire_id)) engine_.pause(*eid);
|
||||
(void)tasks.set_state(wire_id, "paused", std::string(reason));
|
||||
transition(wire_id, "paused", std::string(reason), std::nullopt);
|
||||
}
|
||||
|
||||
std::vector<vdm::TaskId> order;
|
||||
@@ -281,14 +292,27 @@ store::DbResult<void> Scheduler::tick() {
|
||||
return {};
|
||||
}
|
||||
|
||||
void Scheduler::on_engine_state(const std::string& wire_id, std::string_view engine_state,
|
||||
void Scheduler::transition(const std::string& wire_id, std::string_view to_state,
|
||||
std::optional<std::string> pause_reason,
|
||||
const std::optional<TaskErrorFields>& err) {
|
||||
store::Tasks tasks(db_);
|
||||
// pause_reason: an engine-initiated pause carries an error => 'auto' (ADR 0013 §2);
|
||||
// otherwise set_state clears the column.
|
||||
std::optional<std::string> reason;
|
||||
if (engine_state == "paused" && err) reason = "auto";
|
||||
(void)tasks.set_state(wire_id, engine_state, reason);
|
||||
const auto before = tasks.get(wire_id);
|
||||
const std::string previous = (before && before->has_value()) ? (**before).state : std::string();
|
||||
|
||||
// An engine-initiated pause carries an error => 'auto' (ADR 0013 §2), overriding
|
||||
// whatever the caller passed (a scheduler-driven pause never carries an error here).
|
||||
std::optional<std::string> reason = pause_reason;
|
||||
if (to_state == "paused" && err) {
|
||||
reason = "auto";
|
||||
} else if (to_state == "paused" && !reason && before && before->has_value()) {
|
||||
// No reason supplied — the common case is the engine's own pause-ack callback
|
||||
// (on_state(_, paused, nullopt)) arriving after whoever actually initiated the
|
||||
// pause (user_pause() or tick()'s to_pause loop) already wrote the real reason
|
||||
// eagerly. Keep what's already stored instead of clobbering it back to NULL:
|
||||
// set_state() always overwrites the column, reason or not.
|
||||
reason = (**before).pause_reason;
|
||||
}
|
||||
(void)tasks.set_state(wire_id, to_state, reason);
|
||||
|
||||
if (err) {
|
||||
auto st = db_.prepare(
|
||||
@@ -306,12 +330,391 @@ void Scheduler::on_engine_state(const std::string& wire_id, std::string_view eng
|
||||
}
|
||||
}
|
||||
|
||||
if (engine_state == "complete" || engine_state == "failed" || engine_state == "cancelled") {
|
||||
if (!hub_) return;
|
||||
auto after = tasks.get(wire_id);
|
||||
if (!after || !after->has_value()) return;
|
||||
const proto::TaskSummary summary = store::to_summary(**after);
|
||||
|
||||
nlohmann::json params{
|
||||
{"taskId", wire_id},
|
||||
{"state", std::string(to_state)},
|
||||
{"previousState", previous.empty() ? nlohmann::json(nullptr) : nlohmann::json(previous)},
|
||||
{"summary", summary},
|
||||
{"error", summary.error.has_value() ? nlohmann::json(*summary.error) : nlohmann::json(nullptr)},
|
||||
};
|
||||
hub_->publish(proto::Event::TaskState, proto::make_notification(proto::Event::TaskState, params),
|
||||
wire_id);
|
||||
}
|
||||
|
||||
void Scheduler::on_engine_state(const std::string& wire_id, std::string_view from_state,
|
||||
std::string_view to_state,
|
||||
const std::optional<TaskErrorFields>& err) {
|
||||
(void)from_state; // transition() reads the store's own current state as previousState,
|
||||
// which is authoritative regardless of engine/store timing
|
||||
transition(wire_id, to_state, std::nullopt, err);
|
||||
|
||||
if (to_state == "complete" || to_state == "failed" || to_state == "cancelled") {
|
||||
if (auto eid = engine_id_of(wire_id)) {
|
||||
// One last snapshot before the handle goes away: a task that never lived past
|
||||
// a single tick (small/fast/local) would otherwise leave downloadedBytes and
|
||||
// segmentDetail at their pre-segmentation defaults forever, in violation of
|
||||
// TaskDetail.segmentDetail's "exactly summary.segments entries" contract.
|
||||
if (const auto p = engine_.progress(*eid)) persist_progress(wire_id, *p);
|
||||
engine_.release(*eid);
|
||||
unmap_engine(*eid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
vdm::task::DownloadCallbacks Scheduler::make_callbacks(const std::string& wire_id) {
|
||||
vdm::task::DownloadCallbacks cbs;
|
||||
const std::string id_copy = wire_id;
|
||||
|
||||
cbs.on_state = [this, id_copy](vdm::task::EngineState from, vdm::task::EngineState to,
|
||||
const std::optional<vdm::ErrorInfo>& err) {
|
||||
std::optional<TaskErrorFields> ef;
|
||||
if (err) ef = to_error_fields(*err);
|
||||
const std::string from_name = engine_state_name(from);
|
||||
const std::string to_name = engine_state_name(to);
|
||||
deps_.post_to_loop([this, id_copy, from_name, to_name, ef]() {
|
||||
on_engine_state(id_copy, from_name, to_name, ef);
|
||||
});
|
||||
};
|
||||
|
||||
cbs.on_finished = [this, id_copy](vdm::Result<vdm::task::DownloadOutcome> outcome) {
|
||||
deps_.post_to_loop([this, id_copy, outcome]() { on_engine_finished(id_copy, outcome); });
|
||||
};
|
||||
|
||||
return cbs;
|
||||
}
|
||||
|
||||
void Scheduler::on_probe_result(const std::string& wire_id,
|
||||
const vdm::Result<vdm::net::ProbeResult>& pr) {
|
||||
auto got = store::Tasks(db_).get(wire_id);
|
||||
if (!got || !got->has_value()) return; // removed while the probe was outstanding
|
||||
const store::TaskRow& row = **got;
|
||||
|
||||
if (!pr) {
|
||||
transition(wire_id, "failed", std::nullopt, to_error_fields(pr.error()));
|
||||
return;
|
||||
}
|
||||
|
||||
store::Tasks::ProbeFields fields;
|
||||
if (pr->total_size) fields.size_bytes = static_cast<std::int64_t>(*pr->total_size);
|
||||
fields.resumable = pr->resumable;
|
||||
if (!pr->etag.empty()) fields.etag = pr->etag;
|
||||
if (!pr->last_modified.empty()) fields.last_modified = pr->last_modified;
|
||||
if (!pr->mime.empty()) fields.content_type = pr->mime;
|
||||
if (pr->effective_url != row.url) fields.effective_url = pr->effective_url;
|
||||
(void)store::Tasks(db_).set_probe_result(wire_id, fields);
|
||||
|
||||
vdm::task::DownloadSpec spec;
|
||||
spec.url = row.url;
|
||||
spec.save_path = row.save_dir + "/" + row.filename;
|
||||
if (row.req_segments) spec.segments = static_cast<std::uint32_t>(*row.req_segments);
|
||||
if (row.req_buffer_bytes) spec.buffer_bytes = static_cast<std::uint64_t>(*row.req_buffer_bytes);
|
||||
if (row.checksum_algo && row.checksum_value) {
|
||||
vdm::task::Checksum ck;
|
||||
ck.hex = *row.checksum_value;
|
||||
if (*row.checksum_algo == "md5") ck.algo = vdm::task::Checksum::Algo::md5;
|
||||
else if (*row.checksum_algo == "sha1") ck.algo = vdm::task::Checksum::Algo::sha1;
|
||||
else if (*row.checksum_algo == "sha512") ck.algo = vdm::task::Checksum::Algo::sha512;
|
||||
else ck.algo = vdm::task::Checksum::Algo::sha256;
|
||||
spec.checksum = ck;
|
||||
}
|
||||
spec.allow_resume = true; // resume from a sidecar if one is beside save_path
|
||||
spec.probe_hint = *pr; // skip a second probe; the engine still revalidates on resume
|
||||
|
||||
const vdm::TaskId engine_id = engine_.start(spec, make_callbacks(wire_id));
|
||||
map(wire_id, engine_id);
|
||||
// State stays `probing`; the engine's own on_state (probe_hint => starts in
|
||||
// `connecting`) drives the next transition through on_engine_state.
|
||||
}
|
||||
|
||||
void Scheduler::on_engine_finished(const std::string& wire_id,
|
||||
const vdm::Result<vdm::task::DownloadOutcome>& outcome) {
|
||||
// The state transition (complete/failed/cancelled) already happened via on_state,
|
||||
// which always precedes on_finished. This only tops up the byte count for a task that
|
||||
// completed before any progress tick ran — otherwise a fast/local/small transfer
|
||||
// reports downloadedBytes: 0 forever despite a byte-correct file on disk.
|
||||
if (outcome) (void)store::Tasks(db_).set_final_bytes(wire_id, static_cast<std::int64_t>(outcome->bytes));
|
||||
}
|
||||
|
||||
void Scheduler::persist_progress(const std::string& wire_id, const vdm::task::Progress& p) {
|
||||
// TaskDetail.segmentDetail is contractually "exactly TaskSummary.segments entries" —
|
||||
// so the count that goes on the wire as `segments` has to be the length of the list
|
||||
// that actually becomes segmentDetail, not effective_segments (budget slots *held*,
|
||||
// per engine_port.hpp; a small file can hold 8 fairness slots while its segmenter
|
||||
// only ever carves 2 ranges). Falls back to effective_segments only before the task
|
||||
// has any ranges yet, so a `probing`/`connecting` task still reports a sane count.
|
||||
const std::int64_t seg_count = !p.segments.empty()
|
||||
? static_cast<std::int64_t>(p.segments.size())
|
||||
: static_cast<std::int64_t>(p.effective_segments);
|
||||
|
||||
(void)store::Tasks(db_).update_progress(wire_id, static_cast<std::int64_t>(p.downloaded),
|
||||
static_cast<std::int64_t>(p.speed_bps), seg_count,
|
||||
static_cast<std::int64_t>(p.effective_buffer_bytes));
|
||||
|
||||
if (!p.segments.empty()) {
|
||||
std::vector<store::SegmentSnapshot> snaps;
|
||||
snaps.reserve(p.segments.size());
|
||||
for (const auto& s : p.segments) {
|
||||
snaps.push_back({s.index, static_cast<std::int64_t>(s.start),
|
||||
static_cast<std::int64_t>(s.end),
|
||||
static_cast<std::int64_t>(s.completed),
|
||||
static_cast<std::int64_t>(s.speed_bps),
|
||||
segment_state_name(s.state)});
|
||||
}
|
||||
(void)store::Segments(db_).replace_all(wire_id, snaps);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<Scheduler::ProgressRow> Scheduler::progress_snapshot() {
|
||||
std::vector<ProgressRow> out;
|
||||
if (to_engine_.empty()) return out;
|
||||
|
||||
out.reserve(to_engine_.size());
|
||||
for (const auto& [wire_id, engine_id] : to_engine_) {
|
||||
const auto p = engine_.progress(engine_id);
|
||||
if (!p) continue;
|
||||
|
||||
ProgressRow row;
|
||||
row.task_id = wire_id;
|
||||
row.downloaded_bytes = p->downloaded;
|
||||
row.speed_bps = p->speed_bps;
|
||||
row.eta_seconds = p->eta_seconds;
|
||||
for (const auto& s : p->segments)
|
||||
row.segments.push_back({s.index, s.completed, s.speed_bps});
|
||||
out.push_back(std::move(row));
|
||||
|
||||
persist_progress(wire_id, *p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
namespace {
|
||||
bool is_terminal_state(const std::string& s) {
|
||||
return s == "complete" || s == "failed" || s == "cancelled";
|
||||
}
|
||||
} // namespace
|
||||
|
||||
rpc::TaskActionPort::Result Scheduler::user_pause(const std::string& wire_id) {
|
||||
store::Tasks tasks(db_);
|
||||
auto got = tasks.get(wire_id);
|
||||
if (!got || !got->has_value()) return {false, false, {}};
|
||||
const store::TaskRow& row = **got;
|
||||
if (is_terminal_state(row.state) || row.state == "paused")
|
||||
return {true, false, row.state};
|
||||
|
||||
if (auto eid = engine_id_of(wire_id)) engine_.pause(*eid);
|
||||
transition(wire_id, "paused", std::string("user"), std::nullopt);
|
||||
return {true, true, "paused"};
|
||||
}
|
||||
|
||||
rpc::TaskActionPort::Result Scheduler::user_resume(const std::string& wire_id) {
|
||||
store::Tasks tasks(db_);
|
||||
auto got = tasks.get(wire_id);
|
||||
if (!got || !got->has_value()) return {false, false, {}};
|
||||
const store::TaskRow& row = **got;
|
||||
if (row.state != "paused") return {true, false, row.state};
|
||||
|
||||
if (auto eid = engine_id_of(wire_id)) {
|
||||
engine_.resume(*eid);
|
||||
transition(wire_id, "connecting", std::nullopt, std::nullopt);
|
||||
return {true, true, "connecting"};
|
||||
}
|
||||
transition(wire_id, "queued", std::nullopt, std::nullopt);
|
||||
return {true, true, "queued"};
|
||||
}
|
||||
|
||||
rpc::TaskActionPort::Result Scheduler::user_start(const std::string& wire_id) {
|
||||
store::Tasks tasks(db_);
|
||||
auto got = tasks.get(wire_id);
|
||||
if (!got || !got->has_value()) return {false, false, {}};
|
||||
const store::TaskRow& row = **got;
|
||||
if (row.state != "paused" && row.state != "new") return {true, false, row.state};
|
||||
|
||||
if (auto eid = engine_id_of(wire_id)) {
|
||||
engine_.resume(*eid);
|
||||
transition(wire_id, "connecting", std::nullopt, std::nullopt);
|
||||
return {true, true, "connecting"};
|
||||
}
|
||||
transition(wire_id, "queued", std::nullopt, std::nullopt);
|
||||
return {true, true, "queued"};
|
||||
}
|
||||
|
||||
rpc::TaskActionPort::Result Scheduler::user_cancel(const std::string& wire_id,
|
||||
bool discard_partial) {
|
||||
store::Tasks tasks(db_);
|
||||
auto got = tasks.get(wire_id);
|
||||
if (!got || !got->has_value()) return {false, false, {}};
|
||||
const store::TaskRow& row = **got;
|
||||
if (is_terminal_state(row.state)) return {true, false, row.state};
|
||||
|
||||
// Same pattern as tick()'s to_pause loop: call the engine (async, no synchronous
|
||||
// effect) and transition the store eagerly so download.get/list are correct the
|
||||
// instant this call returns. The engine's own on_state(_, cancelled, nullopt) +
|
||||
// on_finished arrive later via on_engine_state, which is what actually
|
||||
// release()s/unmaps the handle — never done here.
|
||||
if (auto eid = engine_id_of(wire_id)) engine_.cancel(*eid, discard_partial);
|
||||
transition(wire_id, "cancelled", std::nullopt, std::nullopt);
|
||||
return {true, true, "cancelled"};
|
||||
}
|
||||
|
||||
std::vector<std::string> Scheduler::pause_queue(const std::string& queue_id) {
|
||||
store::Tasks tasks(db_);
|
||||
proto::TaskFilter filter;
|
||||
filter.queueId = queue_id;
|
||||
filter.states = non_terminal_states();
|
||||
// No paging needed: a queue's max_concurrent is <= 32, so "everything non-terminal in
|
||||
// this queue" is never a large page.
|
||||
auto page = tasks.list(filter, std::nullopt, 0, 10000);
|
||||
std::vector<std::string> paused;
|
||||
if (!page) return paused;
|
||||
|
||||
for (const auto& row : page->rows) {
|
||||
if (run_state_of(row.state) != RunState::Running) continue;
|
||||
if (auto eid = engine_id_of(row.task_id)) engine_.pause(*eid);
|
||||
transition(row.task_id, "paused", std::string("queue_stopped"), std::nullopt);
|
||||
paused.push_back(row.task_id);
|
||||
}
|
||||
return paused;
|
||||
}
|
||||
|
||||
bool Scheduler::provide_auth(const std::string& wire_id, const std::string& username,
|
||||
const std::string& password, bool remember) {
|
||||
(void)remember; // not yet wired to the Secret Service anywhere in this build
|
||||
auto eid = engine_id_of(wire_id);
|
||||
if (!eid) return false;
|
||||
engine_.provide_auth(*eid, username, password, remember);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void Scheduler::probe_now(
|
||||
const proto::DownloadProbeParams& params,
|
||||
std::function<void(proto::HandlerResult<proto::DownloadProbeResult>)> done) {
|
||||
vdm::net::ProbeRequest req;
|
||||
req.url = params.url;
|
||||
if (params.headers)
|
||||
for (const auto& [k, v] : *params.headers) req.headers.push_back({k, v});
|
||||
if (params.cookies)
|
||||
for (const auto& c : *params.cookies) req.cookies.push_back({c.name, c.value});
|
||||
if (params.referrer) req.referrer = *params.referrer;
|
||||
if (params.userAgent) req.user_agent = *params.userAgent;
|
||||
|
||||
engine_.probe(req, [this, params, done](vdm::Result<vdm::net::ProbeResult> pr) {
|
||||
deps_.post_to_loop([this, params, done, pr]() {
|
||||
if (!pr) {
|
||||
nlohmann::json data;
|
||||
if (pr.error().http_status != 0) data["httpStatus"] = pr.error().http_status;
|
||||
done(std::unexpected(proto::HandlerError{
|
||||
proto::ErrorCode::ProbeFailed, pr.error().context, data}));
|
||||
return;
|
||||
}
|
||||
|
||||
const std::string filename = vdm::net::suggest_filename(*pr);
|
||||
|
||||
proto::DownloadProbeResult r;
|
||||
r.filename = filename.empty() ? "download.bin" : filename;
|
||||
if (pr->total_size) r.sizeBytes = static_cast<std::int64_t>(*pr->total_size);
|
||||
r.mime = pr->mime;
|
||||
r.resumable = pr->resumable;
|
||||
r.effectiveUrl = pr->effective_url.empty() ? params.url : pr->effective_url;
|
||||
r.suggestedCategoryId = store::Categories(db_).guess_by_extension(r.filename);
|
||||
if (!pr->etag.empty()) r.etag = pr->etag;
|
||||
if (!pr->last_modified.empty()) r.lastModified = pr->last_modified;
|
||||
r.acceptRanges = pr->accept_ranges;
|
||||
if (!pr->redirect_chain.empty()) r.redirectChain = pr->redirect_chain;
|
||||
if (pr->requires_auth) r.requiresAuth = true;
|
||||
|
||||
store::Settings settings(db_);
|
||||
store::Categories categories(db_);
|
||||
if (auto cats = categories.list()) {
|
||||
for (const auto& c : *cats)
|
||||
if (c.categoryId == r.suggestedCategoryId) {
|
||||
r.suggestedSaveDir = c.saveDir;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!r.suggestedSaveDir) r.suggestedSaveDir = settings.get_string("saveTo.defaultDir");
|
||||
|
||||
done(r);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void Scheduler::refresh_url(
|
||||
const std::string& wire_id, const std::string& url,
|
||||
const std::optional<proto::Headers>& headers,
|
||||
const std::optional<std::vector<proto::Cookie>>& cookies,
|
||||
std::function<void(proto::HandlerResult<proto::DownloadRefreshUrlResult>)> done) {
|
||||
auto got = store::Tasks(db_).get(wire_id);
|
||||
if (!got || !got->has_value()) {
|
||||
done(std::unexpected(proto::HandlerError{proto::ErrorCode::TaskNotFound, "no such task",
|
||||
nlohmann::json{{"taskId", wire_id}}}));
|
||||
return;
|
||||
}
|
||||
const store::TaskRow row = **got; // copied: read again by the async callback below
|
||||
|
||||
vdm::net::ProbeRequest req;
|
||||
req.url = url;
|
||||
if (headers)
|
||||
for (const auto& [k, v] : *headers) req.headers.push_back({k, v});
|
||||
if (cookies)
|
||||
for (const auto& c : *cookies) req.cookies.push_back({c.name, c.value});
|
||||
|
||||
engine_.probe(req, [this, wire_id, row, url, headers, done](vdm::Result<vdm::net::ProbeResult> pr) {
|
||||
deps_.post_to_loop([this, wire_id, row, url, headers, done, pr]() {
|
||||
if (!pr) {
|
||||
nlohmann::json data;
|
||||
if (pr.error().http_status != 0) data["httpStatus"] = pr.error().http_status;
|
||||
done(std::unexpected(proto::HandlerError{proto::ErrorCode::ProbeFailed,
|
||||
pr.error().context, data}));
|
||||
return;
|
||||
}
|
||||
|
||||
// "if they do not [match], it says so rather than silently restarting" (the
|
||||
// schema's own words) — comparison only fires when both sides actually have a
|
||||
// value; an unknown size/validator on either end is not itself a mismatch.
|
||||
bool content_changed = false;
|
||||
if (row.size_bytes && pr->total_size &&
|
||||
*row.size_bytes != static_cast<std::int64_t>(*pr->total_size))
|
||||
content_changed = true;
|
||||
if (row.etag && !row.etag->empty() && !pr->etag.empty() && *row.etag != pr->etag)
|
||||
content_changed = true;
|
||||
if (row.last_modified && !row.last_modified->empty() && !pr->last_modified.empty() &&
|
||||
*row.last_modified != pr->last_modified)
|
||||
content_changed = true;
|
||||
|
||||
store::Tasks tasks(db_);
|
||||
store::Tasks::ProbeFields fields;
|
||||
if (pr->total_size) fields.size_bytes = static_cast<std::int64_t>(*pr->total_size);
|
||||
fields.resumable = pr->resumable;
|
||||
if (!pr->etag.empty()) fields.etag = pr->etag;
|
||||
if (!pr->last_modified.empty()) fields.last_modified = pr->last_modified;
|
||||
if (!pr->mime.empty()) fields.content_type = pr->mime;
|
||||
const std::string effective = pr->effective_url.empty() ? url : pr->effective_url;
|
||||
fields.effective_url = effective;
|
||||
(void)tasks.set_probe_result(wire_id, fields);
|
||||
(void)tasks.set_url(wire_id, url);
|
||||
|
||||
if (auto eid = engine_id_of(wire_id)) {
|
||||
std::vector<vdm::net::HeaderField> hdrs;
|
||||
if (headers)
|
||||
for (const auto& [k, v] : *headers) hdrs.push_back({k, v});
|
||||
engine_.refresh_url(*eid, url, hdrs);
|
||||
}
|
||||
|
||||
proto::DownloadRefreshUrlResult r;
|
||||
r.ok = true;
|
||||
r.resumable = pr->resumable;
|
||||
r.contentChanged = content_changed;
|
||||
if (pr->total_size) r.sizeBytes = static_cast<std::int64_t>(*pr->total_size);
|
||||
r.effectiveUrl = effective;
|
||||
done(r);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::sched
|
||||
|
||||
+139
-13
@@ -4,13 +4,17 @@
|
||||
// wire-UUID <-> vdm::TaskId map and is the only thing that calls EnginePort::start /
|
||||
// pause / resume / set_task_order.
|
||||
//
|
||||
// Threading: tick(), reload_config(), reconcile_after_restart() and the on_engine_*
|
||||
// callbacks all run on ONE thread (the RPC loop). Engine callbacks arrive on engine
|
||||
// threads, so the real wiring passes a `post_to_loop` that marshals them here; the
|
||||
// default runs them inline (tests, single-threaded).
|
||||
// Threading: tick(), reload_config(), reconcile_after_restart(), progress_snapshot() and
|
||||
// the on_engine_* callbacks all run on ONE thread (the RPC loop). Engine callbacks arrive
|
||||
// on engine threads, so the real wiring passes a `post_to_loop` that marshals them here;
|
||||
// the default runs them inline (tests, single-threaded).
|
||||
//
|
||||
// Not yet wired into veloxd — that plus the real EnginePort land when velox::core's
|
||||
// stage-8 bodies reach main (daemon/docs/deferrals.md D4).
|
||||
// event.task.state (D5): on_engine_state publishes it when a hub is supplied — the same
|
||||
// callback that keeps the store row current also keeps subscribed clients current.
|
||||
// event.task.progress is NOT published here: it must be batched into one array message at
|
||||
// <=4 Hz (event.task.progress.schema.json x-maxRateHz), so the caller collects
|
||||
// progress_snapshot() on its own 250 ms timer and does one hub_.publish() with the whole
|
||||
// array, never one per task.
|
||||
|
||||
#include <cstdint>
|
||||
#include <ctime>
|
||||
@@ -18,11 +22,17 @@
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/task_action_port.hpp"
|
||||
#include "sched/engine_port.hpp"
|
||||
#include "sched/governor.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "vdm/ids.hpp"
|
||||
#include "vdm/net/probe.hpp"
|
||||
#include "vdm/task/download.hpp"
|
||||
#include "vdm/util/result.hpp"
|
||||
|
||||
namespace velox::daemon::sched {
|
||||
|
||||
@@ -36,7 +46,12 @@ struct TaskErrorFields {
|
||||
std::optional<std::int64_t> attempt;
|
||||
};
|
||||
|
||||
class Scheduler {
|
||||
// Implements rpc::TaskActionPort directly — sched/ already depends on rpc/ (EventHub), so
|
||||
// this costs nothing new, and it's what lets dispatcher.hpp depend on the port interface
|
||||
// instead of on sched/scheduler.hpp (see rpc/task_action_port.hpp's top comment for why
|
||||
// that matters: it would otherwise make veloxd_rpc <-> veloxd_sched a circular library
|
||||
// dependency).
|
||||
class Scheduler final : public rpc::TaskActionPort {
|
||||
public:
|
||||
// `local_now` returns a fully-populated std::tm in local time; injected so tests can
|
||||
// pin the clock. `post_to_loop` marshals an engine-thread callback onto the loop
|
||||
@@ -46,7 +61,10 @@ public:
|
||||
std::function<void(std::function<void()>)> post_to_loop;
|
||||
};
|
||||
|
||||
Scheduler(store::Db& db, EnginePort& engine, Governor governor, Deps deps = {});
|
||||
// `hub` is optional so unit tests can build a Scheduler with no event fan-out at all;
|
||||
// production wiring always supplies one.
|
||||
Scheduler(store::Db& db, EnginePort& engine, Governor governor, rpc::EventHub* hub = nullptr,
|
||||
Deps deps = {});
|
||||
|
||||
// ADR 0013 §5: on daemon start, every task whose persisted state is a CORE-owned one
|
||||
// (probing..verifying) is rewritten to `queued`; `paused` keeps its pauseReason. The
|
||||
@@ -62,11 +80,87 @@ public:
|
||||
// (start / resume / pause via the engine, update task state rows, push set_task_order).
|
||||
store::DbResult<void> tick();
|
||||
|
||||
// Engine lifecycle callback -> store projection, so the next tick sees ground truth.
|
||||
// Keyed by wire UUID (known when the callback is built, before start() returns the
|
||||
// TaskId). Also the hook the event.task.state fan-out will use — D5.
|
||||
void on_engine_state(const std::string& wire_id, std::string_view engine_state,
|
||||
const std::optional<TaskErrorFields>& err);
|
||||
// Engine lifecycle callback -> store projection, so the next tick sees ground truth,
|
||||
// and (when a hub was supplied) the event.task.state publish. Keyed by wire UUID
|
||||
// (known when the callback is built, before start() returns the TaskId).
|
||||
void on_engine_state(const std::string& wire_id, std::string_view from_state,
|
||||
std::string_view to_state, const std::optional<TaskErrorFields>& err);
|
||||
|
||||
// One row per task the engine is currently tracking, for the caller's
|
||||
// event.task.progress batch. Also writes downloaded_bytes / eff_segments /
|
||||
// eff_buffer_bytes back to the store so download.list / download.get stay current
|
||||
// between state transitions.
|
||||
struct ProgressRow {
|
||||
std::string task_id;
|
||||
std::uint64_t downloaded_bytes;
|
||||
std::uint64_t speed_bps;
|
||||
std::optional<std::uint32_t> eta_seconds;
|
||||
struct Segment {
|
||||
std::uint32_t index;
|
||||
std::uint64_t downloaded_bytes;
|
||||
std::uint64_t speed_bps;
|
||||
};
|
||||
std::vector<Segment> segments;
|
||||
};
|
||||
std::vector<ProgressRow> progress_snapshot();
|
||||
|
||||
// rpc::TaskActionPort. These apply immediately — never wait for the next tick() —
|
||||
// because pausing, resuming or cancelling a live transfer cannot wait up to 1s for the
|
||||
// timerfd, and the governor will never do any of them on its own for a user-owned
|
||||
// reason (ADR 0013 §3: "never touch a task paused for a reason it does not own").
|
||||
// Idempotent: calling one on a task already in the target (or a terminal) state
|
||||
// reports found=true, changed=false.
|
||||
rpc::TaskActionPort::Result user_pause(const std::string& wire_id) override;
|
||||
// A task still holding a live engine handle (paused mid-flight) is engine_.resume()'d
|
||||
// straight back to `connecting`; one with no handle yet (parked since download.add
|
||||
// with startMode 'later', or never admitted) goes to `queued` for the next tick's
|
||||
// normal admission.
|
||||
rpc::TaskActionPort::Result user_resume(const std::string& wire_id) override;
|
||||
// "Begin or restart the given tasks" (download.start): same effect as user_resume for
|
||||
// a paused/new task. NOTE: the contract's "a task in 'queued' jumps its queue" priority
|
||||
// bump is not implemented — admission is still plain FIFO via the governor's
|
||||
// created_at rank. Flagged in deferrals.md.
|
||||
rpc::TaskActionPort::Result user_start(const std::string& wire_id) override;
|
||||
// download.cancel == cancel(discard_partial=false); download.remove == cancel(true)
|
||||
// plus the store row / file cleanup (that part is still D3).
|
||||
rpc::TaskActionPort::Result user_cancel(const std::string& wire_id,
|
||||
bool discard_partial) override;
|
||||
|
||||
// queue.stop(pauseRunning=true): pause every task in `queue_id` the governor would
|
||||
// currently call Running, right now rather than waiting for the next tick — the same
|
||||
// immediacy reasoning as the user_* actions above, with PauseReason::QueueStopped
|
||||
// instead of User. Returns the wire ids actually paused.
|
||||
std::vector<std::string> pause_queue(const std::string& queue_id) override;
|
||||
|
||||
bool provide_auth(const std::string& wire_id, const std::string& username,
|
||||
const std::string& password, bool remember) override;
|
||||
|
||||
// rpc::TaskActionPort::apply_settings_reload — a void-returning wrapper around the
|
||||
// already-public reload_config() above (which returns DbResult<void>, consumed by
|
||||
// main.cpp and by sched_scheduler_test; kept as-is rather than changed to match the
|
||||
// port, which has no caller that wants the DbError).
|
||||
void apply_settings_reload() override { (void)reload_config(); }
|
||||
|
||||
void set_global_speed_limit(std::uint64_t bps) override { engine_.set_global_speed_limit(bps); }
|
||||
|
||||
// rpc::TaskActionPort. Builds a vdm::net::ProbeRequest from `params`, runs it on the
|
||||
// engine's probe pool (outside the segment budget, ADR 0011 §5), and converts the
|
||||
// result back to proto terms — including the suggestedCategoryId/-SaveDir guess (a
|
||||
// plain extension match against the categories table; the real rules engine is D3).
|
||||
// `done` is called already marshalled onto the loop thread via post_to_loop, same as
|
||||
// every other engine callback here — the caller never has to know it started on an
|
||||
// engine thread.
|
||||
void probe_now(
|
||||
const velox::proto::DownloadProbeParams& params,
|
||||
std::function<void(velox::proto::HandlerResult<velox::proto::DownloadProbeResult>)>
|
||||
done) override;
|
||||
|
||||
void refresh_url(
|
||||
const std::string& wire_id, const std::string& url,
|
||||
const std::optional<velox::proto::Headers>& headers,
|
||||
const std::optional<std::vector<velox::proto::Cookie>>& cookies,
|
||||
std::function<void(velox::proto::HandlerResult<velox::proto::DownloadRefreshUrlResult>)>
|
||||
done) override;
|
||||
|
||||
// Diagnostics / tests.
|
||||
std::optional<std::string> wire_id_of(vdm::TaskId id) const;
|
||||
@@ -76,9 +170,41 @@ private:
|
||||
void map(const std::string& wire_id, vdm::TaskId engine_id);
|
||||
void unmap_engine(vdm::TaskId engine_id);
|
||||
|
||||
// The one place a task's state row changes and (if a hub is set) event.task.state
|
||||
// publishes. previousState is read from the store's own current row, not passed in —
|
||||
// authoritative regardless of engine/scheduler timing.
|
||||
void transition(const std::string& wire_id, std::string_view to_state,
|
||||
std::optional<std::string> pause_reason,
|
||||
const std::optional<TaskErrorFields>& err);
|
||||
|
||||
// The probe issued for `wire_id` in tick() has resolved: persist sizeBytes /
|
||||
// resumable / validator, then start() with the result as probe_hint. A probe failure
|
||||
// (bad URL, DNS, 404 with no mirrors) moves the task straight to `failed` — it never
|
||||
// reaches start().
|
||||
void on_probe_result(const std::string& wire_id, const vdm::Result<vdm::net::ProbeResult>& pr);
|
||||
|
||||
// on_finished fired: top up the final byte count (set_final_bytes) so a task that
|
||||
// completed before any progress tick ran still reports real numbers. The state
|
||||
// transition itself (complete/failed/cancelled) already happened via on_engine_state,
|
||||
// which on_finished always follows.
|
||||
void on_engine_finished(const std::string& wire_id,
|
||||
const vdm::Result<vdm::task::DownloadOutcome>& outcome);
|
||||
|
||||
// Writes one task's byte counters + segment rows from a Progress snapshot. Shared by
|
||||
// progress_snapshot() (the periodic tick) and on_engine_state's terminal path (a final
|
||||
// snapshot before release/unmap) so a task that finishes between two ticks — the
|
||||
// common case for anything small or fast — still leaves a real segmentDetail behind
|
||||
// instead of the pre-segmentation empty array.
|
||||
void persist_progress(const std::string& wire_id, const vdm::task::Progress& p);
|
||||
|
||||
// Wires on_state -> on_engine_state and on_finished -> on_engine_finished, both
|
||||
// marshalled through post_to_loop. Shared by the one place a task actually starts.
|
||||
vdm::task::DownloadCallbacks make_callbacks(const std::string& wire_id);
|
||||
|
||||
store::Db& db_;
|
||||
EnginePort& engine_;
|
||||
Governor governor_;
|
||||
rpc::EventHub* hub_;
|
||||
Deps deps_;
|
||||
|
||||
std::unordered_map<std::string, vdm::TaskId> to_engine_;
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
#include "store/categories.hpp"
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cstdio>
|
||||
#include <random>
|
||||
#include <string>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
namespace proto = velox::proto;
|
||||
|
||||
namespace {
|
||||
|
||||
proto::Category project_row(Stmt& st) {
|
||||
proto::Category c;
|
||||
c.categoryId = st.column_text(0);
|
||||
c.name = st.column_text(1);
|
||||
c.saveDir = st.column_text(2);
|
||||
auto j = nlohmann::json::parse(st.column_text(3), nullptr, false);
|
||||
if (j.is_array()) {
|
||||
for (const auto& e : j)
|
||||
if (e.is_string()) c.extensions.push_back(e.get<std::string>());
|
||||
}
|
||||
c.builtin = st.column_int(4) != 0;
|
||||
return c;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
DbResult<std::vector<proto::Category>> Categories::list() {
|
||||
auto st = db_.prepare(
|
||||
"SELECT category_id, name, save_dir, extensions, builtin FROM categories "
|
||||
"ORDER BY builtin DESC, name");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
|
||||
std::vector<proto::Category> out;
|
||||
for (;;) {
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) break;
|
||||
out.push_back(project_row(*st));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
DbResult<std::optional<proto::Category>> Categories::get(std::string_view category_id) {
|
||||
auto st = db_.prepare(
|
||||
"SELECT category_id, name, save_dir, extensions, builtin FROM categories "
|
||||
"WHERE category_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, category_id); !b) return std::unexpected(b.error());
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) return std::optional<proto::Category>{};
|
||||
return std::optional<proto::Category>{project_row(*st)};
|
||||
}
|
||||
|
||||
std::string Categories::guess_by_extension(std::string_view filename_or_ext) {
|
||||
std::string ext(filename_or_ext);
|
||||
if (const auto dot = ext.find_last_of('.'); dot != std::string::npos) ext = ext.substr(dot + 1);
|
||||
if (ext.empty()) return "general";
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(),
|
||||
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
|
||||
|
||||
auto cats = list();
|
||||
if (!cats) return "general";
|
||||
for (const auto& c : *cats) {
|
||||
for (const auto& e : c.extensions)
|
||||
if (e == ext) return c.categoryId;
|
||||
}
|
||||
return "general";
|
||||
}
|
||||
|
||||
DbResult<proto::Category> Categories::upsert(proto::Category category) {
|
||||
// A replace keeps the existing row's builtin flag; a create is never builtin. Either
|
||||
// way the payload's own `builtin` is ignored — a client cannot mint or revoke it.
|
||||
bool builtin = false;
|
||||
if (!category.categoryId.empty()) {
|
||||
auto existing = get(category.categoryId);
|
||||
if (!existing) return std::unexpected(existing.error());
|
||||
if (existing->has_value()) builtin = (*existing)->builtin;
|
||||
} else {
|
||||
// Reuse Tasks' id scheme (v4 UUID) would need a cross-module include for one
|
||||
// function; a category id has no wire format requirement beyond "a string", so a
|
||||
// timestamp-free random hex id keeps this module self-contained.
|
||||
std::random_device rd;
|
||||
std::uniform_int_distribution<std::uint64_t> d;
|
||||
char buf[17];
|
||||
std::snprintf(buf, sizeof(buf), "%016llx", static_cast<unsigned long long>(d(rd)));
|
||||
category.categoryId = std::string(buf);
|
||||
}
|
||||
|
||||
nlohmann::json ext = nlohmann::json::array();
|
||||
for (const auto& e : category.extensions) ext.push_back(e);
|
||||
|
||||
auto st = db_.prepare(
|
||||
"INSERT INTO categories(category_id, name, save_dir, extensions, builtin) "
|
||||
"VALUES(?1,?2,?3,?4,?5) "
|
||||
"ON CONFLICT(category_id) DO UPDATE SET "
|
||||
"name=excluded.name, save_dir=excluded.save_dir, extensions=excluded.extensions");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, std::string_view(category.categoryId)); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto b = st->bind(2, std::string_view(category.name)); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(3, std::string_view(category.saveDir)); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto b = st->bind(4, std::string_view(ext.dump())); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(5, static_cast<std::int64_t>(builtin)); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
|
||||
category.builtin = builtin;
|
||||
return category;
|
||||
}
|
||||
|
||||
DbResult<Categories::RemoveResult> Categories::remove(
|
||||
std::string_view category_id, const std::optional<std::string>& reassign_to) {
|
||||
RemoveResult out;
|
||||
const std::string target = reassign_to.value_or("general");
|
||||
|
||||
// Db::transaction only threads a DbResult<void> lambda; `out` is filled in-place and
|
||||
// returned once the transaction (which may still fail and roll back) succeeds.
|
||||
auto txn = db_.transaction([&]() -> DbResult<void> {
|
||||
{
|
||||
// A builtin category is never removed, and — since it was never going to be
|
||||
// removed — its tasks must not be reassigned away from it either.
|
||||
auto chk = db_.prepare("SELECT builtin FROM categories WHERE category_id = ?1");
|
||||
if (!chk) return std::unexpected(chk.error());
|
||||
if (auto b = chk->bind(1, category_id); !b) return std::unexpected(b.error());
|
||||
auto row = chk->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) return {}; // no such category: removed stays false
|
||||
if (chk->column_int(0) != 0) return {}; // builtin: removed stays false
|
||||
}
|
||||
{
|
||||
auto sel = db_.prepare("SELECT task_id FROM tasks WHERE category_id = ?1");
|
||||
if (!sel) return std::unexpected(sel.error());
|
||||
if (auto b = sel->bind(1, category_id); !b) return std::unexpected(b.error());
|
||||
for (;;) {
|
||||
auto row = sel->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) break;
|
||||
out.reassigned_task_ids.push_back(sel->column_text(0));
|
||||
}
|
||||
}
|
||||
if (!out.reassigned_task_ids.empty()) {
|
||||
auto upd = db_.prepare("UPDATE tasks SET category_id = ?2 WHERE category_id = ?1");
|
||||
if (!upd) return std::unexpected(upd.error());
|
||||
if (auto b = upd->bind(1, category_id); !b) return std::unexpected(b.error());
|
||||
if (auto b = upd->bind(2, std::string_view(target)); !b) return std::unexpected(b.error());
|
||||
if (auto r = upd->step(); !r) return std::unexpected(r.error());
|
||||
}
|
||||
auto del = db_.prepare("DELETE FROM categories WHERE category_id = ?1 AND builtin = 0");
|
||||
if (!del) return std::unexpected(del.error());
|
||||
if (auto b = del->bind(1, category_id); !b) return std::unexpected(b.error());
|
||||
if (auto r = del->step(); !r) return std::unexpected(r.error());
|
||||
out.removed = sqlite3_changes(db_.raw()) > 0;
|
||||
return {};
|
||||
});
|
||||
if (!txn) return std::unexpected(txn.error());
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,56 @@
|
||||
#pragma once
|
||||
|
||||
// Read access to the `categories` table, projected onto proto::Category. Owned here
|
||||
// rather than duplicated per handler since category.list and download.add (rule
|
||||
// matching, later) both need it.
|
||||
//
|
||||
// The table has no columns for Category.mimeTypes / .sortOrder (0001_initial.sql predates
|
||||
// those fields); upsert() accepts them but they are not persisted — round-tripped as unset
|
||||
// on the next list()/get(). Noted in daemon/docs/deferrals.md.
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "store/sqlite.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
class Categories {
|
||||
public:
|
||||
explicit Categories(Db& db) : db_(db) {}
|
||||
|
||||
DbResult<std::vector<velox::proto::Category>> list();
|
||||
DbResult<std::optional<velox::proto::Category>> get(std::string_view category_id);
|
||||
|
||||
// Extension match against categories.extensions — not the real rules engine (no
|
||||
// host/mime/size clauses), just enough that a capture or a File Info preselect isn't
|
||||
// always "general". `filename_or_ext` may be a whole filename ("movie.mp4") or a bare
|
||||
// extension ("mp4", no leading dot); matched case-insensitively. "general" (this
|
||||
// project's always-present default category) on no match, an empty/dotless filename,
|
||||
// or a store error — this never fails outward, it just falls back.
|
||||
std::string guess_by_extension(std::string_view filename_or_ext);
|
||||
|
||||
// "Omit categoryId to create; supply it to replace" (category.upsert's own words) —
|
||||
// the caller (dispatcher) decides create vs replace by whether `category.categoryId`
|
||||
// is empty and generates the id; this just writes the row. `builtin` is never taken
|
||||
// from the payload: preserved from the existing row on a replace, always false on a
|
||||
// create (a client can never mint a builtin category).
|
||||
DbResult<velox::proto::Category> upsert(velox::proto::Category category);
|
||||
|
||||
// False for "no such category". A builtin category is never removed — the caller
|
||||
// checks that (category.remove -> -32602) before calling this, since that check needs
|
||||
// ErrorCode, which this module (like the rest of store/) does not depend on.
|
||||
struct RemoveResult {
|
||||
bool removed = false;
|
||||
std::vector<std::string> reassigned_task_ids;
|
||||
};
|
||||
DbResult<RemoveResult> remove(std::string_view category_id,
|
||||
const std::optional<std::string>& reassign_to);
|
||||
|
||||
private:
|
||||
Db& db_;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,30 @@
|
||||
-- Migration 0002 — segments gets a speed_bps column and a corrected state CHECK;
|
||||
-- tasks gets a speed_bps column too.
|
||||
--
|
||||
-- 0001's segments.state CHECK omitted 'pending' (vdm::segment::SegState::idle's wire
|
||||
-- spelling — "range assigned, no worker connected yet"), so a segment snapshot taken
|
||||
-- before its first worker connects could never be written. SQLite cannot ALTER a CHECK
|
||||
-- constraint in place, so this rebuilds the table (standard SQLite pattern: create the
|
||||
-- new shape, copy, drop, rename). speed_bps on both tables backs TaskSummary.speedBps /
|
||||
-- Segment.speedBps on the wire; absent from 0001 because progress writing wasn't wired
|
||||
-- yet — a task's engine-reported aggregate speed had nowhere to persist between polls.
|
||||
|
||||
ALTER TABLE tasks ADD COLUMN speed_bps INTEGER NOT NULL DEFAULT 0;
|
||||
|
||||
CREATE TABLE segments_new (
|
||||
task_id TEXT NOT NULL REFERENCES tasks(task_id) ON DELETE CASCADE,
|
||||
idx INTEGER NOT NULL,
|
||||
start_byte INTEGER NOT NULL,
|
||||
end_byte INTEGER NOT NULL,
|
||||
completed_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
speed_bps INTEGER NOT NULL DEFAULT 0,
|
||||
state TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (state IN ('pending','connecting','downloading','stalled','complete','failed')),
|
||||
PRIMARY KEY (task_id, idx)
|
||||
) STRICT, WITHOUT ROWID;
|
||||
|
||||
INSERT INTO segments_new (task_id, idx, start_byte, end_byte, completed_bytes, state)
|
||||
SELECT task_id, idx, start_byte, end_byte, completed_bytes, state FROM segments;
|
||||
|
||||
DROP TABLE segments;
|
||||
ALTER TABLE segments_new RENAME TO segments;
|
||||
@@ -0,0 +1,91 @@
|
||||
-- Migration 0003 — tasks.start_mode is rebuilt to the contract's StartMode values.
|
||||
--
|
||||
-- 0001's CHECK read `start_mode IN ('auto','now','queue','manual')`. That is not
|
||||
-- StartMode.schema.json's enum at all: the contract is ['now','later','queue']. The
|
||||
-- practical effect: `download.add` with `startMode: "later"` — a real, documented value
|
||||
-- (the File Info dialog's Download Later button) — hit the CHECK constraint on insert
|
||||
-- and surfaced as an unhandled -32603, every time. 'auto' and 'manual' were never
|
||||
-- contract values; they were this table's own invention and nothing on the wire ever
|
||||
-- sends them.
|
||||
--
|
||||
-- SQLite cannot ALTER a CHECK constraint in place, so this rebuilds the table (same
|
||||
-- pattern as 0002: create the new shape, copy with the value mapped, drop, rename).
|
||||
-- Existing rows are remapped by what they actually meant: 'auto' was "eligible for the
|
||||
-- scheduler the moment it's added", i.e. 'now'; 'manual' was "parked, wait for the user",
|
||||
-- which is what 'later' means on the wire (StartMode's own description: "lands the task
|
||||
-- in paused"). Any row already spelled 'now' or 'queue' passes through unchanged.
|
||||
--
|
||||
-- This does NOT touch `state` or `pause_reason` — a row that was start_mode='manual' and
|
||||
-- (per the dispatcher's now-dead branch) state='new' keeps state='new'; the daemon-side
|
||||
-- fix to actually land a 'later' task in 'paused' going forward lives in dispatcher.cpp,
|
||||
-- not in this migration. Historical rows are not replayed through the scheduler.
|
||||
|
||||
CREATE TABLE tasks_new (
|
||||
task_id TEXT PRIMARY KEY,
|
||||
url TEXT NOT NULL,
|
||||
effective_url TEXT,
|
||||
filename TEXT NOT NULL DEFAULT '',
|
||||
save_dir TEXT NOT NULL,
|
||||
category_id TEXT REFERENCES categories(category_id) ON DELETE SET NULL,
|
||||
queue_id TEXT REFERENCES queues(queue_id) ON DELETE SET NULL,
|
||||
queue_position INTEGER,
|
||||
|
||||
state TEXT NOT NULL DEFAULT 'new'
|
||||
CHECK (state IN ('new','probing','queued','connecting','downloading','paused',
|
||||
'retry_wait','assembling','verifying','complete','failed','cancelled')),
|
||||
pause_reason TEXT CHECK (pause_reason IN
|
||||
('user','schedule','queue_stopped','admission_reconcile','auto')),
|
||||
|
||||
size_bytes INTEGER,
|
||||
downloaded_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
resumable INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
req_segments INTEGER,
|
||||
eff_segments INTEGER NOT NULL DEFAULT 0,
|
||||
req_buffer_bytes INTEGER,
|
||||
eff_buffer_bytes INTEGER,
|
||||
|
||||
-- The contract's StartMode (StartMode.schema.json): 'now' | 'later' | 'queue'.
|
||||
start_mode TEXT NOT NULL DEFAULT 'now'
|
||||
CHECK (start_mode IN ('now','later','queue')),
|
||||
description TEXT,
|
||||
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
content_type TEXT,
|
||||
|
||||
checksum_algo TEXT CHECK (checksum_algo IN ('md5','sha1','sha256','sha512')),
|
||||
checksum_value TEXT,
|
||||
|
||||
error_code TEXT,
|
||||
error_message TEXT,
|
||||
error_http_status INTEGER,
|
||||
error_retryable INTEGER,
|
||||
error_attempt INTEGER,
|
||||
error_next_retry_at TEXT,
|
||||
|
||||
speed_bps INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
created_at TEXT NOT NULL,
|
||||
last_try_at TEXT,
|
||||
completed_at TEXT
|
||||
) STRICT;
|
||||
|
||||
INSERT INTO tasks_new
|
||||
SELECT task_id, url, effective_url, filename, save_dir, category_id, queue_id,
|
||||
queue_position, state, pause_reason, size_bytes, downloaded_bytes, resumable,
|
||||
req_segments, eff_segments, req_buffer_bytes, eff_buffer_bytes,
|
||||
CASE start_mode WHEN 'auto' THEN 'now' WHEN 'manual' THEN 'later' ELSE start_mode END,
|
||||
description, etag, last_modified, content_type, checksum_algo, checksum_value,
|
||||
error_code, error_message, error_http_status, error_retryable, error_attempt,
|
||||
error_next_retry_at, speed_bps, created_at, last_try_at, completed_at
|
||||
FROM tasks;
|
||||
|
||||
DROP TABLE tasks;
|
||||
ALTER TABLE tasks_new RENAME TO tasks;
|
||||
|
||||
CREATE INDEX idx_tasks_state ON tasks(state);
|
||||
CREATE INDEX idx_tasks_category ON tasks(category_id);
|
||||
CREATE INDEX idx_tasks_queue_order ON tasks(queue_id, queue_position);
|
||||
CREATE INDEX idx_tasks_created ON tasks(created_at);
|
||||
CREATE INDEX idx_tasks_completed ON tasks(completed_at);
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Migration 0004 — rules gets a name column.
|
||||
--
|
||||
-- 0001's rules table had no column for Rule.name (Rule.schema.json's own optional,
|
||||
-- maxLength-64 label field) — store/rules.cpp discovered this the hard way building
|
||||
-- rules.list/rules.upsert: every rules.list call failed outright ("no such column:
|
||||
-- name") because the SELECT it needs to project onto proto::Rule names a column that was
|
||||
-- never there. A plain ALTER TABLE ADD COLUMN suffices here (no CHECK constraint to
|
||||
-- rebuild around, unlike 0002/0003).
|
||||
|
||||
ALTER TABLE rules ADD COLUMN name TEXT;
|
||||
@@ -0,0 +1,183 @@
|
||||
#include "store/queues.hpp"
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <random>
|
||||
#include <string>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
namespace proto = velox::proto;
|
||||
|
||||
namespace {
|
||||
|
||||
// One queue row (columns queue_id, name, state, max_concurrent, schedule, on_complete, in
|
||||
// that order) plus its member taskIds, read off the row a caller has already step()'d to.
|
||||
DbResult<proto::Queue> project_row(Db& db, Stmt& st) {
|
||||
proto::Queue q;
|
||||
q.queueId = st.column_text(0);
|
||||
q.name = st.column_text(1);
|
||||
if (auto s = proto::parse_QueueState(st.column_text(2))) q.state = *s;
|
||||
q.maxConcurrent = st.column_int(3);
|
||||
if (!st.column_is_null(4)) {
|
||||
auto j = nlohmann::json::parse(st.column_text(4), nullptr, false);
|
||||
if (auto sched = proto::parse<proto::Schedule>(j, "schedule")) q.schedule = *sched;
|
||||
}
|
||||
if (auto oc = proto::parse_QueueOnComplete(st.column_text(5))) q.onComplete = *oc;
|
||||
|
||||
auto ts = db.prepare("SELECT task_id FROM tasks WHERE queue_id = ?1 ORDER BY queue_position");
|
||||
if (!ts) return std::unexpected(ts.error());
|
||||
if (auto b = ts->bind(1, std::string_view(q.queueId)); !b) return std::unexpected(b.error());
|
||||
std::vector<std::string> ids;
|
||||
for (;;) {
|
||||
auto r = ts->step();
|
||||
if (!r) return std::unexpected(r.error());
|
||||
if (!*r) break;
|
||||
ids.push_back(ts->column_text(0));
|
||||
}
|
||||
q.taskIds = std::move(ids);
|
||||
return q;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
DbResult<std::vector<proto::Queue>> Queues::list() {
|
||||
auto st = db_.prepare(
|
||||
"SELECT queue_id, name, state, max_concurrent, schedule, on_complete FROM queues "
|
||||
"ORDER BY name");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
|
||||
std::vector<proto::Queue> out;
|
||||
for (;;) {
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) break;
|
||||
auto q = project_row(db_, *st);
|
||||
if (!q) return std::unexpected(q.error());
|
||||
out.push_back(std::move(*q));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
DbResult<std::optional<proto::Queue>> Queues::get(std::string_view queue_id) {
|
||||
auto st = db_.prepare(
|
||||
"SELECT queue_id, name, state, max_concurrent, schedule, on_complete FROM queues "
|
||||
"WHERE queue_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, queue_id); !b) return std::unexpected(b.error());
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) return std::optional<proto::Queue>{};
|
||||
auto q = project_row(db_, *st);
|
||||
if (!q) return std::unexpected(q.error());
|
||||
return std::optional<proto::Queue>{std::move(*q)};
|
||||
}
|
||||
|
||||
DbResult<bool> Queues::set_state(std::string_view queue_id, std::string_view state) {
|
||||
auto st = db_.prepare("UPDATE queues SET state = ?2 WHERE queue_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, queue_id); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(2, state); !b) return std::unexpected(b.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Queues::set_schedule(std::string_view queue_id,
|
||||
const std::optional<proto::Schedule>& schedule) {
|
||||
auto st = db_.prepare("UPDATE queues SET schedule = ?2 WHERE queue_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, queue_id); !b) return std::unexpected(b.error());
|
||||
if (auto r = schedule ? st->bind(2, std::string_view(nlohmann::json(*schedule).dump()))
|
||||
: st->bind_null(2);
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Queues::reorder(std::string_view queue_id, const std::vector<std::string>& task_ids) {
|
||||
std::vector<std::string> current;
|
||||
{
|
||||
auto st = db_.prepare(
|
||||
"SELECT task_id FROM tasks WHERE queue_id = ?1 ORDER BY queue_position");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, queue_id); !b) return std::unexpected(b.error());
|
||||
for (;;) {
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) break;
|
||||
current.push_back(st->column_text(0));
|
||||
}
|
||||
}
|
||||
|
||||
// Exact permutation: same size, same members, order aside.
|
||||
std::vector<std::string> a = current, b = task_ids;
|
||||
std::sort(a.begin(), a.end());
|
||||
std::sort(b.begin(), b.end());
|
||||
if (a != b) return false;
|
||||
|
||||
auto txn = db_.transaction([&]() -> DbResult<void> {
|
||||
for (std::size_t i = 0; i < task_ids.size(); ++i) {
|
||||
auto st = db_.prepare("UPDATE tasks SET queue_position = ?2 WHERE task_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto bd = st->bind(1, std::string_view(task_ids[i])); !bd)
|
||||
return std::unexpected(bd.error());
|
||||
if (auto bd = st->bind(2, static_cast<std::int64_t>(i)); !bd)
|
||||
return std::unexpected(bd.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
}
|
||||
return {};
|
||||
});
|
||||
if (!txn) return std::unexpected(txn.error());
|
||||
return true;
|
||||
}
|
||||
|
||||
DbResult<proto::Queue> Queues::upsert(proto::Queue queue) {
|
||||
if (queue.queueId.empty()) {
|
||||
std::random_device rd;
|
||||
std::uniform_int_distribution<std::uint64_t> d;
|
||||
char buf[17];
|
||||
std::snprintf(buf, sizeof(buf), "%016llx", static_cast<unsigned long long>(d(rd)));
|
||||
queue.queueId = std::string(buf);
|
||||
}
|
||||
|
||||
// A create defaults to 'stopped' (never auto-runs a brand-new queue); a replace keeps
|
||||
// whatever run state the queue is already in — queue.upsert edits the config, not the
|
||||
// run state (that's queue.start/stop).
|
||||
std::string state = "stopped";
|
||||
if (auto existing = get(queue.queueId); existing && existing->has_value())
|
||||
state = std::string(proto::to_string((*existing)->state));
|
||||
|
||||
const std::string schedule_json =
|
||||
queue.schedule ? nlohmann::json(*queue.schedule).dump() : std::string();
|
||||
const std::string on_complete =
|
||||
std::string(proto::to_string(queue.onComplete.value_or(proto::QueueOnComplete::Nothing)));
|
||||
|
||||
auto st = db_.prepare(
|
||||
"INSERT INTO queues(queue_id, name, state, max_concurrent, schedule, on_complete) "
|
||||
"VALUES(?1,?2,?3,?4,?5,?6) "
|
||||
"ON CONFLICT(queue_id) DO UPDATE SET "
|
||||
"name=excluded.name, max_concurrent=excluded.max_concurrent, "
|
||||
"schedule=excluded.schedule, on_complete=excluded.on_complete");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, std::string_view(queue.queueId)); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(2, std::string_view(queue.name)); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(3, std::string_view(state)); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(4, queue.maxConcurrent); !b) return std::unexpected(b.error());
|
||||
if (auto r = queue.schedule ? st->bind(5, std::string_view(schedule_json)) : st->bind_null(5); !r)
|
||||
return std::unexpected(r.error());
|
||||
if (auto b = st->bind(6, std::string_view(on_complete)); !b) return std::unexpected(b.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
|
||||
auto stored = get(queue.queueId);
|
||||
if (!stored) return std::unexpected(stored.error());
|
||||
if (!stored->has_value())
|
||||
return std::unexpected(DbError{0, "queue.upsert: row vanished after insert"});
|
||||
return **stored;
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,52 @@
|
||||
#pragma once
|
||||
|
||||
// Read access to the `queues` table, projected onto proto::Queue. taskIds is derived from
|
||||
// `tasks` (queue_id = this queue, ordered by queue_position), not stored on the queue row
|
||||
// — membership changes through download.update / queue.reorder, per Queue's own schema
|
||||
// note that a queue.upsert payload's taskIds is ignored.
|
||||
|
||||
#include <optional>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "store/sqlite.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
class Queues {
|
||||
public:
|
||||
explicit Queues(Db& db) : db_(db) {}
|
||||
|
||||
DbResult<std::vector<velox::proto::Queue>> list();
|
||||
|
||||
// nullopt (not an error) if no queue has this id.
|
||||
DbResult<std::optional<velox::proto::Queue>> get(std::string_view queue_id);
|
||||
|
||||
// 'running' or 'stopped' (Queue.schema.json / the state column's CHECK). false if the
|
||||
// id doesn't exist.
|
||||
DbResult<bool> set_state(std::string_view queue_id, std::string_view state);
|
||||
|
||||
// schedule.set: nullopt clears it (NULL = manual control, per the schema's own
|
||||
// words). false if the queue doesn't exist.
|
||||
DbResult<bool> set_schedule(std::string_view queue_id,
|
||||
const std::optional<velox::proto::Schedule>& schedule);
|
||||
|
||||
// queue.reorder: `task_ids` must be an exact permutation of the queue's current
|
||||
// membership (the schema's own words — "anything else is -32602 rather than a
|
||||
// partial reorder, so a stale drag from an out-of-date view cannot quietly reshuffle
|
||||
// the queue"). false (no write at all) if it isn't; true and queue_position rewritten
|
||||
// to match `task_ids`'s order if it is.
|
||||
DbResult<bool> reorder(std::string_view queue_id, const std::vector<std::string>& task_ids);
|
||||
|
||||
// "Omit queueId to create" (queue.upsert's own words) — an empty id generates one.
|
||||
// taskIds is ignored (membership changes only through download.update / queue.reorder,
|
||||
// per the schema's own note); a create defaults to 'stopped', a replace keeps the
|
||||
// queue's current run state (queue.upsert edits config, not run state).
|
||||
DbResult<velox::proto::Queue> upsert(velox::proto::Queue queue);
|
||||
|
||||
private:
|
||||
Db& db_;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,96 @@
|
||||
#include "store/rules.hpp"
|
||||
|
||||
#include <cstdio>
|
||||
#include <random>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
namespace proto = velox::proto;
|
||||
|
||||
namespace {
|
||||
|
||||
proto::Rule project_row(Stmt& st) {
|
||||
proto::Rule r;
|
||||
r.ruleId = st.column_text(0);
|
||||
if (!st.column_is_null(1)) r.name = st.column_text(1);
|
||||
r.enabled = st.column_int(2) != 0;
|
||||
r.priority = st.column_int(3);
|
||||
if (auto j = nlohmann::json::parse(st.column_text(4), nullptr, false); !j.is_discarded()) {
|
||||
if (auto m = proto::parse<proto::RuleMatch>(j, "match")) r.match = *m;
|
||||
}
|
||||
if (auto j = nlohmann::json::parse(st.column_text(5), nullptr, false); !j.is_discarded()) {
|
||||
if (auto a = proto::parse<proto::RuleAction>(j, "action")) r.action = *a;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
std::string new_rule_id() {
|
||||
std::random_device rd;
|
||||
std::uniform_int_distribution<std::uint64_t> d;
|
||||
char buf[17];
|
||||
std::snprintf(buf, sizeof(buf), "%016llx", static_cast<unsigned long long>(d(rd)));
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
DbResult<std::vector<proto::Rule>> Rules::list() {
|
||||
auto st = db_.prepare(
|
||||
"SELECT rule_id, name, enabled, priority, match, action FROM rules "
|
||||
"ORDER BY priority, rule_id");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
|
||||
std::vector<proto::Rule> out;
|
||||
for (;;) {
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) break;
|
||||
out.push_back(project_row(*st));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
DbResult<std::vector<proto::Rule>> Rules::apply(std::vector<proto::Rule> upsert,
|
||||
const std::vector<std::string>& remove) {
|
||||
auto txn = db_.transaction([&]() -> DbResult<void> {
|
||||
for (const auto& id : remove) {
|
||||
auto del = db_.prepare("DELETE FROM rules WHERE rule_id = ?1");
|
||||
if (!del) return std::unexpected(del.error());
|
||||
if (auto b = del->bind(1, std::string_view(id)); !b) return std::unexpected(b.error());
|
||||
if (auto r = del->step(); !r) return std::unexpected(r.error());
|
||||
}
|
||||
for (auto& rule : upsert) {
|
||||
if (rule.ruleId.empty()) rule.ruleId = new_rule_id();
|
||||
|
||||
nlohmann::json match_json = rule.match;
|
||||
nlohmann::json action_json = rule.action;
|
||||
|
||||
auto st = db_.prepare(
|
||||
"INSERT INTO rules(rule_id, name, enabled, priority, match, action) "
|
||||
"VALUES(?1,?2,?3,?4,?5,?6) "
|
||||
"ON CONFLICT(rule_id) DO UPDATE SET "
|
||||
"name=excluded.name, enabled=excluded.enabled, priority=excluded.priority, "
|
||||
"match=excluded.match, action=excluded.action");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, std::string_view(rule.ruleId)); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto r = rule.name ? st->bind(2, std::string_view(*rule.name)) : st->bind_null(2); !r)
|
||||
return std::unexpected(r.error());
|
||||
if (auto b = st->bind(3, static_cast<std::int64_t>(rule.enabled)); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto b = st->bind(4, rule.priority); !b) return std::unexpected(b.error());
|
||||
if (auto b = st->bind(5, std::string_view(match_json.dump())); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto b = st->bind(6, std::string_view(action_json.dump())); !b)
|
||||
return std::unexpected(b.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
}
|
||||
return {};
|
||||
});
|
||||
if (!txn) return std::unexpected(txn.error());
|
||||
return list();
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,37 @@
|
||||
#pragma once
|
||||
|
||||
// Read/write access to the `rules` table (rules.list / rules.upsert / capture.offer's own
|
||||
// read path). match/action are stored as their generated-JSON text (velox::proto::RuleMatch
|
||||
// / RuleAction), so no bespoke schema lives here beyond the table's own columns.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "store/sqlite.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
class Rules {
|
||||
public:
|
||||
explicit Rules(Db& db) : db_(db) {}
|
||||
|
||||
// Enabled and disabled rules alike, in priority order (ties broken by rule_id) —
|
||||
// rules.list's own contract ("the rules engine's table, in priority order"); capture
|
||||
// offer's own caller filters to enabled ones itself, same as vdm::rules::match_rules
|
||||
// already does internally.
|
||||
DbResult<std::vector<velox::proto::Rule>> list();
|
||||
|
||||
// rules.upsert: "upsert carries the rules to store and remove the ruleIds to drop;
|
||||
// applying both at once means a reprioritisation never leaves the table in a
|
||||
// half-valid state" (the schema's own words) — one transaction. An empty ruleId in
|
||||
// `upsert` generates one (create); a non-empty one replaces. Returns the full table
|
||||
// after the write, in priority order.
|
||||
DbResult<std::vector<velox::proto::Rule>> apply(std::vector<velox::proto::Rule> upsert,
|
||||
const std::vector<std::string>& remove);
|
||||
|
||||
private:
|
||||
Db& db_;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,61 @@
|
||||
#include "store/segments.hpp"
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
namespace proto = velox::proto;
|
||||
|
||||
DbResult<void> Segments::replace_all(std::string_view task_id,
|
||||
const std::vector<SegmentSnapshot>& segs) {
|
||||
return db_.transaction([&]() -> DbResult<void> {
|
||||
{
|
||||
auto del = db_.prepare("DELETE FROM segments WHERE task_id = ?1");
|
||||
if (!del) return std::unexpected(del.error());
|
||||
if (auto r = del->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = del->step(); !r) return std::unexpected(r.error());
|
||||
}
|
||||
for (const auto& s : segs) {
|
||||
auto ins = db_.prepare(
|
||||
"INSERT INTO segments(task_id, idx, start_byte, end_byte, completed_bytes, "
|
||||
"speed_bps, state) VALUES(?1,?2,?3,?4,?5,?6,?7)");
|
||||
if (!ins) return std::unexpected(ins.error());
|
||||
if (auto r = ins->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = ins->bind(2, static_cast<std::int64_t>(s.index)); !r)
|
||||
return std::unexpected(r.error());
|
||||
if (auto r = ins->bind(3, s.start_byte); !r) return std::unexpected(r.error());
|
||||
if (auto r = ins->bind(4, s.end_byte); !r) return std::unexpected(r.error());
|
||||
if (auto r = ins->bind(5, s.completed_bytes); !r) return std::unexpected(r.error());
|
||||
if (auto r = ins->bind(6, s.speed_bps); !r) return std::unexpected(r.error());
|
||||
if (auto r = ins->bind(7, std::string_view(s.state)); !r)
|
||||
return std::unexpected(r.error());
|
||||
if (auto r = ins->step(); !r) return std::unexpected(r.error());
|
||||
}
|
||||
return {};
|
||||
});
|
||||
}
|
||||
|
||||
DbResult<std::vector<proto::Segment>> Segments::list(std::string_view task_id) {
|
||||
auto st = db_.prepare(
|
||||
"SELECT idx, start_byte, end_byte, completed_bytes, speed_bps, state "
|
||||
"FROM segments WHERE task_id = ?1 ORDER BY idx");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto r = st->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
|
||||
std::vector<proto::Segment> out;
|
||||
for (;;) {
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
if (!*row) break;
|
||||
|
||||
proto::Segment seg;
|
||||
seg.index = st->column_int(0);
|
||||
seg.startByte = st->column_int(1);
|
||||
seg.endByte = st->column_int(2);
|
||||
seg.downloadedBytes = st->column_int(3);
|
||||
seg.speedBps = st->column_int(4);
|
||||
if (auto s = proto::parse_SegmentState(st->column_text(5))) seg.state = *s;
|
||||
out.push_back(seg);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -0,0 +1,45 @@
|
||||
#pragma once
|
||||
|
||||
// Read/write access to the `segments` table — the per-connection detail behind
|
||||
// TaskDetail.segmentDetail (download.get) and the segment-bars widget. Written from an
|
||||
// engine progress tick (Scheduler::progress_snapshot); read back on download.get.
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
#include "store/sqlite.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
namespace velox::daemon::store {
|
||||
|
||||
// One segment's live counters, as read off vdm::task::SegmentProgress. `state` is a wire
|
||||
// SegmentState spelling ("pending"/"connecting"/.../"failed") — the caller maps the
|
||||
// engine's own enum, this module stays generic like the others.
|
||||
struct SegmentSnapshot {
|
||||
std::uint32_t index;
|
||||
std::int64_t start_byte;
|
||||
std::int64_t end_byte; // inclusive (ADR 0010)
|
||||
std::int64_t completed_bytes;
|
||||
std::int64_t speed_bps;
|
||||
std::string state;
|
||||
};
|
||||
|
||||
class Segments {
|
||||
public:
|
||||
explicit Segments(Db& db) : db_(db) {}
|
||||
|
||||
// Replaces every row for `task_id` with `segs` in one transaction. Simpler than a
|
||||
// per-index upsert and cheap at <=32 rows, <=4 Hz.
|
||||
DbResult<void> replace_all(std::string_view task_id, const std::vector<SegmentSnapshot>& segs);
|
||||
|
||||
// In index order. Empty if the task has never been segmented (TaskDetail's own
|
||||
// description permits this).
|
||||
DbResult<std::vector<velox::proto::Segment>> list(std::string_view task_id);
|
||||
|
||||
private:
|
||||
Db& db_;
|
||||
};
|
||||
|
||||
} // namespace velox::daemon::store
|
||||
@@ -9,24 +9,62 @@ namespace velox::daemon::store {
|
||||
|
||||
namespace {
|
||||
|
||||
// Built-in defaults, mirroring Settings.schema.json / ADR 0012. Only the keys the daemon
|
||||
// currently reads or is likely to need before the full settings.get handler lands; the
|
||||
// rest resolve through the schema's own defaults at that layer.
|
||||
constexpr std::array<std::pair<std::string_view, std::string_view>, 14> kDefaults{{
|
||||
// Built-in defaults, one per SettingKey (Settings.schema.json / SettingKey.schema.json).
|
||||
// The schema itself carries no "default" keyword anywhere — these are what settings.get
|
||||
// falls back to for a key with no stored row, chosen per ADR 0012 where it speaks (the
|
||||
// connection.* buffer/segment keys) and otherwise the conservative, least-surprising
|
||||
// value for that key's own description. capture.monitoredExtensions defaults to the union
|
||||
// of every builtin category's extensions (0001_initial.sql's seed) rather than an
|
||||
// arbitrary list of its own, so the two stay in sync without a second place to edit.
|
||||
constexpr std::array<std::pair<std::string_view, std::string_view>, 43> kDefaults{{
|
||||
{"general.launchOnLogin", "false"},
|
||||
{"general.minimizeToTray", "false"},
|
||||
{"general.showDropTarget", "true"},
|
||||
{"general.confirmOnExit", "true"},
|
||||
{"general.language", "\"system\""},
|
||||
{"general.checkForUpdates", "true"},
|
||||
{"capture.enabled", "true"},
|
||||
{"capture.monitoredExtensions",
|
||||
"[\"exe\",\"msi\",\"deb\",\"rpm\",\"dmg\",\"appimage\",\"iso\",\"zip\",\"tar\",\"gz\","
|
||||
"\"xz\",\"7z\",\"mp4\",\"mkv\",\"webm\",\"avi\",\"mov\",\"flv\",\"m4v\",\"ts\",\"mp3\","
|
||||
"\"flac\",\"aac\",\"ogg\",\"opus\",\"wav\",\"m4a\",\"pdf\",\"doc\",\"docx\",\"xls\","
|
||||
"\"xlsx\",\"ppt\",\"pptx\",\"odt\",\"epub\",\"jpg\",\"jpeg\",\"png\",\"gif\",\"webp\","
|
||||
"\"svg\",\"bmp\",\"tiff\"]"},
|
||||
{"capture.monitoredMimeTypes", "[]"},
|
||||
{"capture.minSizeBytes", "0"},
|
||||
{"capture.excludedHosts", "[]"},
|
||||
{"capture.bypassModifier", "\"shift\""},
|
||||
{"capture.autoStartTypes", "[]"},
|
||||
{"saveTo.defaultDir", "\"~/Downloads\""},
|
||||
{"saveTo.tempDir", "\"\""},
|
||||
{"saveTo.allowedRoots", "[\"~/Downloads\"]"},
|
||||
{"saveTo.fileExistsPolicy", "\"ask\""},
|
||||
{"saveTo.createSubfolderPerSite", "false"},
|
||||
{"connection.preset", "\"auto\""},
|
||||
{"connection.maxSegmentsPerDownload", "8"},
|
||||
{"connection.bufferBytes", "1048576"},
|
||||
{"connection.maxConcurrentDownloads", "5"},
|
||||
{"connection.maxActiveSegments", "32"},
|
||||
{"connection.maxTotalBufferBytes", "134217728"},
|
||||
{"connection.maxActiveSegments", "32"},
|
||||
{"connection.maxConcurrentDownloads", "5"},
|
||||
{"connection.timeoutSec", "30"},
|
||||
{"connection.maxRetries", "10"},
|
||||
{"connection.retryBackoffSec", "5"},
|
||||
{"saveTo.defaultDir", "\"~/Downloads\""},
|
||||
{"saveTo.allowedRoots", "[\"~/Downloads\"]"},
|
||||
{"saveTo.createSubfolderPerSite", "false"},
|
||||
{"downloads.speedLimitBps", "0"},
|
||||
{"downloads.speedLimitEnabled", "false"},
|
||||
{"downloads.virusScanCommand", "\"\""},
|
||||
{"downloads.postDownloadCommand", "\"\""},
|
||||
{"downloads.duplicatePolicy", "\"ask\""},
|
||||
{"downloads.verifyChecksums", "true"},
|
||||
{"proxy.mode", "\"system\""},
|
||||
{"proxy.host", "\"\""},
|
||||
{"proxy.port", "1"},
|
||||
{"proxy.username", "\"\""},
|
||||
{"proxy.bypassHosts", "[]"},
|
||||
{"proxy.pacUrl", "\"\""},
|
||||
{"sounds.enabled", "true"},
|
||||
{"sounds.onComplete", "\"\""},
|
||||
{"sounds.onQueueComplete", "\"\""},
|
||||
{"sounds.onError", "\"\""},
|
||||
}};
|
||||
|
||||
} // namespace
|
||||
@@ -86,6 +124,19 @@ std::int64_t Settings::get_int(std::string_view key) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool Settings::get_bool(std::string_view key) {
|
||||
auto raw = get_raw(key);
|
||||
if (raw && *raw) {
|
||||
auto j = nlohmann::json::parse(**raw, nullptr, false);
|
||||
if (j.is_boolean()) return j.get<bool>();
|
||||
}
|
||||
if (auto d = default_for(key)) {
|
||||
auto j = nlohmann::json::parse(*d, nullptr, false);
|
||||
if (j.is_boolean()) return j.get<bool>();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string Settings::get_string(std::string_view key) {
|
||||
auto raw = get_raw(key);
|
||||
if (raw && *raw) {
|
||||
|
||||
@@ -35,6 +35,7 @@ public:
|
||||
// Typed convenience over get_raw + the defaults. A malformed stored value falls back
|
||||
// to the default rather than throwing.
|
||||
std::int64_t get_int(std::string_view key);
|
||||
bool get_bool(std::string_view key);
|
||||
std::string get_string(std::string_view key);
|
||||
std::vector<std::string> get_string_array(std::string_view key);
|
||||
|
||||
|
||||
+152
-6
@@ -19,7 +19,7 @@ constexpr const char* kCols =
|
||||
"size_bytes, downloaded_bytes, resumable, "
|
||||
"req_segments, eff_segments, req_buffer_bytes, eff_buffer_bytes, queue_position, "
|
||||
"error_code, error_message, error_http_status, error_retryable, error_attempt, "
|
||||
"error_next_retry_at";
|
||||
"error_next_retry_at, speed_bps";
|
||||
|
||||
DbResult<void> bind_opt(Stmt& s, int i, const std::optional<std::string>& v) {
|
||||
return v ? s.bind(i, std::string_view(*v)) : s.bind_null(i);
|
||||
@@ -72,6 +72,7 @@ TaskRow read_row(Stmt& s) {
|
||||
if (!s.column_is_null(30)) r.error_retryable = s.column_int(30) != 0;
|
||||
r.error_attempt = col_opt_int(s, 31);
|
||||
r.error_next_retry_at = col_opt_text(s, 32);
|
||||
r.speed_bps = s.column_int(33);
|
||||
return r;
|
||||
}
|
||||
|
||||
@@ -103,9 +104,9 @@ DbResult<void> Tasks::insert(const TaskRow& r) {
|
||||
"checksum_algo, checksum_value, size_bytes, downloaded_bytes, resumable, "
|
||||
"req_segments, eff_segments, req_buffer_bytes, eff_buffer_bytes, queue_position, "
|
||||
"error_code, error_message, error_http_status, error_retryable, error_attempt, "
|
||||
"error_next_retry_at) VALUES("
|
||||
"error_next_retry_at, speed_bps) VALUES("
|
||||
"?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19,?20,?21,?22,"
|
||||
"?23,?24,?25,?26,?27,?28,?29,?30,?31,?32,?33)");
|
||||
"?23,?24,?25,?26,?27,?28,?29,?30,?31,?32,?33,?34)");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
|
||||
auto chk = [](DbResult<void> r) { return r.has_value(); };
|
||||
@@ -134,7 +135,8 @@ DbResult<void> Tasks::insert(const TaskRow& r) {
|
||||
chk(r.error_retryable ? st->bind(31, static_cast<std::int64_t>(*r.error_retryable))
|
||||
: st->bind_null(31)) &&
|
||||
chk(bind_opt(*st, 32, r.error_attempt)) &&
|
||||
chk(bind_opt(*st, 33, r.error_next_retry_at));
|
||||
chk(bind_opt(*st, 33, r.error_next_retry_at)) &&
|
||||
chk(st->bind(34, r.speed_bps));
|
||||
if (!ok) return std::unexpected(DbError{0, "failed to bind a task column"});
|
||||
|
||||
if (auto r2 = st->step(); !r2) return std::unexpected(r2.error());
|
||||
@@ -264,6 +266,50 @@ DbResult<bool> Tasks::remove(std::string_view task_id) {
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Tasks::update_progress(std::string_view task_id, std::int64_t downloaded_bytes,
|
||||
std::int64_t speed_bps, std::int64_t eff_segments,
|
||||
std::int64_t eff_buffer_bytes) {
|
||||
auto st = db_.prepare(
|
||||
"UPDATE tasks SET downloaded_bytes = ?2, speed_bps = ?3, eff_segments = ?4, "
|
||||
"eff_buffer_bytes = ?5 WHERE task_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto r = st->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(2, downloaded_bytes); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(3, speed_bps); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(4, eff_segments); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(5, eff_buffer_bytes); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Tasks::set_probe_result(std::string_view task_id, const ProbeFields& f) {
|
||||
auto st = db_.prepare(
|
||||
"UPDATE tasks SET size_bytes = ?2, resumable = ?3, etag = ?4, last_modified = ?5, "
|
||||
"content_type = ?6, effective_url = ?7 WHERE task_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto r = st->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = bind_opt(*st, 2, f.size_bytes); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(3, static_cast<std::int64_t>(f.resumable)); !r)
|
||||
return std::unexpected(r.error());
|
||||
if (auto r = bind_opt(*st, 4, f.etag); !r) return std::unexpected(r.error());
|
||||
if (auto r = bind_opt(*st, 5, f.last_modified); !r) return std::unexpected(r.error());
|
||||
if (auto r = bind_opt(*st, 6, f.content_type); !r) return std::unexpected(r.error());
|
||||
if (auto r = bind_opt(*st, 7, f.effective_url); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Tasks::set_final_bytes(std::string_view task_id, std::int64_t bytes) {
|
||||
auto st = db_.prepare(
|
||||
"UPDATE tasks SET downloaded_bytes = ?2, size_bytes = COALESCE(size_bytes, ?2) "
|
||||
"WHERE task_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto r = st->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(2, bytes); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<std::int64_t> Tasks::count() {
|
||||
auto st = db_.prepare("SELECT count(*) FROM tasks");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
@@ -272,6 +318,99 @@ DbResult<std::int64_t> Tasks::count() {
|
||||
return (*row) ? st->column_int(0) : 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Tasks::apply_update(std::string_view task_id, const UpdatePatch& patch) {
|
||||
// One UPDATE per present field: simplest thing that's obviously correct for a
|
||||
// single-row edit with ~9 independent optional fields, and it means a field the
|
||||
// caller didn't touch is never rewritten with its own unchanged value (matters for
|
||||
// no-op-detection callers, though download.update doesn't currently need that).
|
||||
bool touched_any = false;
|
||||
auto run = [&](const char* sql, auto&& binder) -> DbResult<void> {
|
||||
auto st = db_.prepare(sql);
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto r = st->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = binder(*st); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
touched_any = true;
|
||||
return {};
|
||||
};
|
||||
|
||||
if (patch.save_dir && patch.filename) {
|
||||
if (auto r = run("UPDATE tasks SET save_dir = ?2, filename = ?3 WHERE task_id = ?1",
|
||||
[&](Stmt& s) {
|
||||
if (auto b = s.bind(2, std::string_view(*patch.save_dir)); !b) return b;
|
||||
return s.bind(3, std::string_view(*patch.filename));
|
||||
});
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
if (patch.category_id) {
|
||||
if (auto r = run("UPDATE tasks SET category_id = ?2 WHERE task_id = ?1",
|
||||
[&](Stmt& s) { return s.bind(2, std::string_view(*patch.category_id)); });
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
if (patch.queue_id) {
|
||||
if (auto r = run("UPDATE tasks SET queue_id = ?2, queue_position = ?3 WHERE task_id = ?1",
|
||||
[&](Stmt& s) {
|
||||
if (auto b = s.bind(2, std::string_view(*patch.queue_id)); !b) return b;
|
||||
return patch.queue_position ? s.bind(3, *patch.queue_position)
|
||||
: s.bind_null(3);
|
||||
});
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
if (patch.description) {
|
||||
if (auto r = run("UPDATE tasks SET description = ?2 WHERE task_id = ?1",
|
||||
[&](Stmt& s) { return s.bind(2, std::string_view(*patch.description)); });
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
if (patch.req_segments) {
|
||||
if (auto r = run("UPDATE tasks SET req_segments = ?2 WHERE task_id = ?1",
|
||||
[&](Stmt& s) { return s.bind(2, *patch.req_segments); });
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
if (patch.req_buffer_bytes) {
|
||||
if (auto r = run("UPDATE tasks SET req_buffer_bytes = ?2 WHERE task_id = ?1",
|
||||
[&](Stmt& s) { return s.bind(2, *patch.req_buffer_bytes); });
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
if (patch.checksum_algo && patch.checksum_value) {
|
||||
if (auto r = run(
|
||||
"UPDATE tasks SET checksum_algo = ?2, checksum_value = ?3 WHERE task_id = ?1",
|
||||
[&](Stmt& s) {
|
||||
if (auto b = s.bind(2, std::string_view(*patch.checksum_algo)); !b) return b;
|
||||
return s.bind(3, std::string_view(*patch.checksum_value));
|
||||
});
|
||||
!r)
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
|
||||
return touched_any;
|
||||
}
|
||||
|
||||
DbResult<bool> Tasks::set_url(std::string_view task_id, std::string_view url) {
|
||||
auto st = db_.prepare("UPDATE tasks SET url = ?2 WHERE task_id = ?1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto r = st->bind(1, task_id); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->bind(2, url); !r) return std::unexpected(r.error());
|
||||
if (auto r = st->step(); !r) return std::unexpected(r.error());
|
||||
return sqlite3_changes(db_.raw()) > 0;
|
||||
}
|
||||
|
||||
DbResult<bool> Tasks::has_active_duplicate(std::string_view url) {
|
||||
auto st = db_.prepare(
|
||||
"SELECT 1 FROM tasks WHERE url = ?1 "
|
||||
"AND state NOT IN ('complete','failed','cancelled') LIMIT 1");
|
||||
if (!st) return std::unexpected(st.error());
|
||||
if (auto b = st->bind(1, url); !b) return std::unexpected(b.error());
|
||||
auto row = st->step();
|
||||
if (!row) return std::unexpected(row.error());
|
||||
return *row;
|
||||
}
|
||||
|
||||
proto::TaskSummary to_summary(const TaskRow& r) {
|
||||
proto::TaskSummary s;
|
||||
s.taskId = r.task_id;
|
||||
@@ -282,9 +421,16 @@ proto::TaskSummary to_summary(const TaskRow& r) {
|
||||
s.sizeBytes = r.size_bytes;
|
||||
s.downloadedBytes = r.downloaded_bytes;
|
||||
if (auto st = proto::parse_TaskState(r.state)) s.state = *st;
|
||||
s.speedBps = 0;
|
||||
s.speedBps = r.speed_bps;
|
||||
s.resumable = r.resumable;
|
||||
s.segments = r.eff_segments;
|
||||
// TaskSummary.segments is minimum:1, always -- even a task that has never connected
|
||||
// reports the count it WOULD use (its requested value, or the frozen default),
|
||||
// never the "not started yet" placeholder of 0 that used to leak onto the wire.
|
||||
s.segments = r.eff_segments > 0 ? r.eff_segments
|
||||
: r.req_segments && *r.req_segments > 0 ? *r.req_segments
|
||||
: 8;
|
||||
if (s.segments < 1) s.segments = 1;
|
||||
if (s.segments > 32) s.segments = 32;
|
||||
s.categoryId = r.category_id;
|
||||
s.queueId = r.queue_id;
|
||||
s.queuePosition = r.queue_position;
|
||||
|
||||
@@ -21,7 +21,7 @@ struct TaskRow {
|
||||
std::string save_dir;
|
||||
std::string filename;
|
||||
std::string state = "new";
|
||||
std::string start_mode = "auto";
|
||||
std::string start_mode = "now"; // contract StartMode: 'now'|'later'|'queue'
|
||||
std::string created_at;
|
||||
|
||||
std::optional<std::string> effective_url;
|
||||
@@ -39,6 +39,7 @@ struct TaskRow {
|
||||
|
||||
std::optional<std::int64_t> size_bytes;
|
||||
std::int64_t downloaded_bytes = 0;
|
||||
std::int64_t speed_bps = 0;
|
||||
bool resumable = false;
|
||||
|
||||
std::optional<std::int64_t> req_segments;
|
||||
@@ -78,6 +79,64 @@ public:
|
||||
DbResult<bool> remove(std::string_view task_id);
|
||||
DbResult<std::int64_t> count();
|
||||
|
||||
// capture.offer's dedupe check: true if a non-terminal task already targets this exact
|
||||
// URL (the same rule download.add itself does not enforce — a deliberate re-add is
|
||||
// allowed there; capture is the automatic path where re-grabbing an in-flight download
|
||||
// is almost always a mistake, e.g. two tabs triggering the same link).
|
||||
DbResult<bool> has_active_duplicate(std::string_view url);
|
||||
|
||||
// download.update's patch, already resolved by the caller (new save_dir/filename
|
||||
// canonicalized and root-checked, any file already moved on disk — this only writes
|
||||
// the row). Every field is applied when present; queue_position is written alongside
|
||||
// queue_id (nullopt leaves the existing position alone — the caller decides what
|
||||
// "moved into a queue" should set it to). Note: the generated parser collapses "field
|
||||
// absent" and "field explicitly null" to the same nullopt (DownloadUpdateParamsPatch
|
||||
// has no way to tell them apart on the wire as generated), so this — like the RPC
|
||||
// layer above it — can only ever set category_id/queue_id/description/checksum, never
|
||||
// clear them back to NULL through this call.
|
||||
struct UpdatePatch {
|
||||
std::optional<std::string> save_dir;
|
||||
std::optional<std::string> filename;
|
||||
std::optional<std::string> category_id;
|
||||
std::optional<std::string> queue_id;
|
||||
std::optional<std::int64_t> queue_position;
|
||||
std::optional<std::string> description;
|
||||
std::optional<std::int64_t> req_segments;
|
||||
std::optional<std::int64_t> req_buffer_bytes;
|
||||
std::optional<std::string> checksum_algo;
|
||||
std::optional<std::string> checksum_value;
|
||||
};
|
||||
DbResult<bool> apply_update(std::string_view task_id, const UpdatePatch& patch);
|
||||
|
||||
// download.refreshUrl: point the task at a freshly-issued URL. Separate from
|
||||
// apply_update/set_probe_result since neither owns the base `url` column — refreshUrl
|
||||
// is the one caller that changes it after creation.
|
||||
DbResult<bool> set_url(std::string_view task_id, std::string_view url);
|
||||
|
||||
// Byte-counter update from an engine progress tick — cheaper than a full row rewrite,
|
||||
// and keeps download.list / download.get current between state transitions.
|
||||
DbResult<bool> update_progress(std::string_view task_id, std::int64_t downloaded_bytes,
|
||||
std::int64_t speed_bps, std::int64_t eff_segments,
|
||||
std::int64_t eff_buffer_bytes);
|
||||
|
||||
// What the probe learned, persisted before start() so a task that completes before any
|
||||
// progress tick still reports a real sizeBytes / resumable (not the pre-probe default).
|
||||
struct ProbeFields {
|
||||
std::optional<std::int64_t> size_bytes;
|
||||
bool resumable = false;
|
||||
std::optional<std::string> etag;
|
||||
std::optional<std::string> last_modified;
|
||||
std::optional<std::string> content_type;
|
||||
std::optional<std::string> effective_url;
|
||||
};
|
||||
DbResult<bool> set_probe_result(std::string_view task_id, const ProbeFields& fields);
|
||||
|
||||
// on_finished's byte count, for a task that completes before any progress tick ever
|
||||
// ran (see AGENT-DAEMON review: the bug this closes). size_bytes is only filled in if
|
||||
// still unset — the probe's total_size is the more authoritative source when both
|
||||
// exist and happen to disagree (a chunked source with no declared length, say).
|
||||
DbResult<bool> set_final_bytes(std::string_view task_id, std::int64_t bytes);
|
||||
|
||||
private:
|
||||
Db& db_;
|
||||
};
|
||||
|
||||
@@ -20,4 +20,11 @@ veloxd_test(sched_window LIBS veloxd_sched)
|
||||
veloxd_test(sched_governor LIBS veloxd_sched)
|
||||
veloxd_test(safepath LIBS veloxd_fs)
|
||||
veloxd_test(store_tasks LIBS veloxd_store)
|
||||
veloxd_test(sched_scheduler LIBS veloxd_sched)
|
||||
veloxd_test(sched_scheduler LIBS veloxd_sched veloxd_rpc)
|
||||
veloxd_test(event_hub LIBS veloxd_rpc)
|
||||
veloxd_test(store_categories_queues LIBS veloxd_store)
|
||||
veloxd_test(single_instance LIBS veloxd_rpc)
|
||||
veloxd_test(systemd_activation LIBS veloxd_rpc)
|
||||
veloxd_test(dispatcher_settings LIBS veloxd_rpc veloxd_store)
|
||||
veloxd_test(capture_offer LIBS veloxd_rpc veloxd_store)
|
||||
veloxd_test(dispatcher_misc LIBS veloxd_rpc veloxd_store)
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
// capture.offer: excluded hosts, type/size filtering, rule matching, dedupe, take/ignore,
|
||||
// and — the point of this file — the 750 ms deadline actually gets enforced when the
|
||||
// store is slow, without a real sleep anywhere (a fake CaptureDataSource advances its own
|
||||
// clock instead).
|
||||
|
||||
#include <chrono>
|
||||
#include <string>
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/capture_data_source.hpp"
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "store/migrations.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "store/tasks.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
using namespace velox::daemon;
|
||||
namespace proto = velox::proto;
|
||||
|
||||
namespace {
|
||||
|
||||
// A controllable CaptureDataSource: every accessor returns a canned value, and `now()`
|
||||
// reads a clock the test (or a "slow" accessor) can jump forward instantly. No real time
|
||||
// ever passes — a test that jumps 2 real seconds still runs in microseconds.
|
||||
class FakeCaptureSource : public rpc::CaptureDataSource {
|
||||
public:
|
||||
std::chrono::steady_clock::time_point clock = std::chrono::steady_clock::now();
|
||||
bool enabled = true;
|
||||
std::vector<std::string> ext = {"mp4"};
|
||||
std::vector<std::string> mime;
|
||||
std::int64_t min_size = 0;
|
||||
std::vector<std::string> excluded;
|
||||
std::vector<vdm::rules::Rule> rules_;
|
||||
std::string category = "video";
|
||||
std::string save_dir = "~/Downloads/velox-capture-test";
|
||||
bool duplicate = false;
|
||||
|
||||
// Set to jump the clock forward by this much the next time the named accessor is
|
||||
// called — simulates "this particular store read took a long time."
|
||||
std::chrono::milliseconds slow_on_rules{0};
|
||||
std::chrono::milliseconds slow_on_duplicate{0};
|
||||
|
||||
std::chrono::steady_clock::time_point now() override { return clock; }
|
||||
bool capture_enabled() override { return enabled; }
|
||||
std::vector<std::string> monitored_extensions() override { return ext; }
|
||||
std::vector<std::string> monitored_mime_types() override { return mime; }
|
||||
std::int64_t min_size_bytes() override { return min_size; }
|
||||
std::vector<std::string> excluded_hosts() override { return excluded; }
|
||||
std::vector<vdm::rules::Rule> enabled_rules() override {
|
||||
clock += slow_on_rules;
|
||||
return rules_;
|
||||
}
|
||||
std::string guess_category_id(const std::string&) override { return category; }
|
||||
std::string category_save_dir(const std::string&) override { return save_dir; }
|
||||
std::string default_save_dir() override { return save_dir; }
|
||||
bool has_active_duplicate(const std::string&) override {
|
||||
clock += slow_on_duplicate;
|
||||
return duplicate;
|
||||
}
|
||||
};
|
||||
|
||||
proto::CaptureOfferParams offer(std::string url) {
|
||||
proto::CaptureOfferParams p;
|
||||
p.url = std::move(url);
|
||||
p.method = proto::CaptureOfferParamsMethod::GET;
|
||||
p.tabUrl = "https://example.com/page";
|
||||
p.filename = std::string("movie.mp4");
|
||||
p.contentType = std::string("video/mp4");
|
||||
p.contentLength = 5'000'000;
|
||||
return p;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void run() {
|
||||
auto db = store::Db::open(":memory:");
|
||||
CHECK(db.has_value());
|
||||
if (!db) return;
|
||||
CHECK(store::migrate_to_head(*db).has_value());
|
||||
|
||||
rpc::EventHub hub;
|
||||
rpc::VeloxDispatcher dispatcher(*db, hub);
|
||||
|
||||
// --- capture disabled --------------------------------------------------------------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.enabled = false;
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK(r->action == proto::CaptureOfferResultAction::Ignore);
|
||||
CHECK(r->reason == proto::CaptureOfferResultReason::CaptureDisabled);
|
||||
}
|
||||
}
|
||||
|
||||
// --- excluded host -------------------------------------------------------------------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.excluded = {"*.excluded.example"};
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.excluded.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK(r->action == proto::CaptureOfferResultAction::Ignore);
|
||||
CHECK(r->reason == proto::CaptureOfferResultReason::ExcludedHost);
|
||||
}
|
||||
}
|
||||
|
||||
// --- type not monitored --------------------------------------------------------------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.ext = {"iso"}; // movie.mp4 doesn't match, and mime list is empty
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) CHECK(r->reason == proto::CaptureOfferResultReason::TypeNotMonitored);
|
||||
}
|
||||
|
||||
// --- below minimum size ----------------------------------------------------------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.min_size = 10'000'000; // offer's contentLength is 5,000,000
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) CHECK(r->reason == proto::CaptureOfferResultReason::BelowMinSize);
|
||||
}
|
||||
|
||||
// --- a rule says ignore this host --------------------------------------------------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
vdm::rules::Rule rule;
|
||||
rule.rule_id = "r1";
|
||||
rule.enabled = true;
|
||||
rule.priority = 0;
|
||||
rule.match.host_pattern = "cdn.example";
|
||||
rule.action.capture = vdm::rules::CaptureVerdict::ignore;
|
||||
src.rules_ = {rule};
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) CHECK(r->reason == proto::CaptureOfferResultReason::RuleIgnore);
|
||||
}
|
||||
|
||||
// --- duplicate -----------------------------------------------------------------------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.duplicate = true;
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) CHECK(r->reason == proto::CaptureOfferResultReason::Duplicate);
|
||||
}
|
||||
|
||||
// --- take: a real task gets created, saved under the resolved category dir --------
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK(r->action == proto::CaptureOfferResultAction::Take);
|
||||
CHECK(r->taskId.has_value());
|
||||
if (r->taskId) {
|
||||
store::Tasks tasks(*db);
|
||||
auto row = tasks.get(*r->taskId);
|
||||
CHECK(row.has_value() && row->has_value());
|
||||
if (row && *row) {
|
||||
// resolve_target expands "~" and returns the canonical absolute path,
|
||||
// not the literal string handed in.
|
||||
CHECK((*row)->save_dir.find("velox-capture-test") != std::string::npos);
|
||||
CHECK_EQ((*row)->category_id.value_or(""), std::string("video"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- the deadline: a "slow" store still gets an answer, and it's `ignore` --------
|
||||
// slow_on_rules jumps the fake clock forward 2 real seconds' worth the moment
|
||||
// enabled_rules() is read (simulating a slow rules-table read); the 700 ms budget is
|
||||
// long blown by the time the next checkpoint runs, so the offer must be ignored
|
||||
// rather than proceeding to actually take the download. No real time passes — this
|
||||
// whole test runs in microseconds.
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.slow_on_rules = std::chrono::milliseconds(2000);
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
const auto wall_start = std::chrono::steady_clock::now();
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
const auto wall_elapsed = std::chrono::steady_clock::now() - wall_start;
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK(r->action == proto::CaptureOfferResultAction::Ignore);
|
||||
CHECK(!r->taskId.has_value());
|
||||
}
|
||||
// The real wall clock barely moved — only the fake one jumped — proving the
|
||||
// deadline check reads the injected clock, not a real sleep standing in for one.
|
||||
CHECK(wall_elapsed < std::chrono::milliseconds(500));
|
||||
}
|
||||
|
||||
// Same again, but the slow step is the dedupe check instead of rule matching — the
|
||||
// deadline check has to run between every step, not just after one particular call.
|
||||
{
|
||||
FakeCaptureSource src;
|
||||
src.slow_on_duplicate = std::chrono::milliseconds(2000);
|
||||
dispatcher.set_capture_source_for_test(&src);
|
||||
auto r = dispatcher.on_capture_offer(offer("https://cdn.example/movie.mp4"));
|
||||
CHECK(r.has_value());
|
||||
if (r) CHECK(r->action == proto::CaptureOfferResultAction::Ignore);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
@@ -0,0 +1,244 @@
|
||||
// rules.list/upsert, queue.reorder, schedule.get/set, limiter.get/set, download.update:
|
||||
// the rest of D3, called directly against an in-memory store (no socket).
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "store/migrations.hpp"
|
||||
#include "store/queues.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "store/tasks.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
using namespace velox::daemon;
|
||||
namespace proto = velox::proto;
|
||||
|
||||
namespace {
|
||||
|
||||
store::TaskRow task(std::string id, std::optional<std::string> queue = std::nullopt,
|
||||
std::int64_t pos = 0) {
|
||||
store::TaskRow r;
|
||||
r.task_id = std::move(id);
|
||||
r.url = "https://cdn.example/" + r.task_id;
|
||||
r.save_dir = "/tmp";
|
||||
r.filename = r.task_id + ".bin";
|
||||
r.state = "queued";
|
||||
r.created_at = "2026-09-12T00:00:00Z";
|
||||
r.queue_id = std::move(queue);
|
||||
if (r.queue_id) r.queue_position = pos;
|
||||
return r;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void run() {
|
||||
auto db = store::Db::open(":memory:");
|
||||
CHECK(db.has_value());
|
||||
if (!db) return;
|
||||
CHECK(store::migrate_to_head(*db).has_value());
|
||||
|
||||
rpc::EventHub hub;
|
||||
rpc::VeloxDispatcher dispatcher(*db, hub);
|
||||
|
||||
// --- rules.list starts empty; rules.upsert creates, replaces, and removes --------
|
||||
{
|
||||
auto listed = dispatcher.on_rules_list({});
|
||||
CHECK(listed.has_value());
|
||||
if (listed) CHECK(listed->items.empty());
|
||||
|
||||
proto::RulesUpsertParams up;
|
||||
proto::Rule r;
|
||||
r.name = "ISOs to a queue";
|
||||
r.enabled = true;
|
||||
r.priority = 5;
|
||||
r.match.extensions = std::vector<std::string>{"iso"};
|
||||
r.action.categoryId = "programs";
|
||||
up.upsert = {r};
|
||||
auto created = dispatcher.on_rules_upsert(up);
|
||||
CHECK(created.has_value());
|
||||
if (created) {
|
||||
CHECK_EQ(created->items.size(), std::size_t{1});
|
||||
CHECK(!created->items[0].ruleId.empty());
|
||||
}
|
||||
|
||||
// Replace: same ruleId, new priority.
|
||||
if (!created || created->items.empty()) return;
|
||||
std::string rule_id = created->items[0].ruleId;
|
||||
proto::RulesUpsertParams replace;
|
||||
proto::Rule r2 = created->items[0];
|
||||
r2.priority = 1;
|
||||
replace.upsert = {r2};
|
||||
auto replaced = dispatcher.on_rules_upsert(replace);
|
||||
CHECK(replaced.has_value());
|
||||
if (replaced) {
|
||||
CHECK_EQ(replaced->items.size(), std::size_t{1});
|
||||
CHECK_EQ(replaced->items[0].priority, std::int64_t{1});
|
||||
}
|
||||
|
||||
// Remove.
|
||||
proto::RulesUpsertParams rm;
|
||||
rm.remove = {rule_id};
|
||||
auto removed = dispatcher.on_rules_upsert(rm);
|
||||
CHECK(removed.has_value());
|
||||
if (removed) CHECK(removed->items.empty());
|
||||
}
|
||||
|
||||
// --- queue.reorder: a real permutation applies; a non-permutation is -32602 ------
|
||||
{
|
||||
store::Tasks tasks(*db);
|
||||
CHECK(tasks.insert(task("q0", "main", 0)).has_value());
|
||||
CHECK(tasks.insert(task("q1", "main", 1)).has_value());
|
||||
CHECK(tasks.insert(task("q2", "main", 2)).has_value());
|
||||
|
||||
proto::QueueReorderParams p;
|
||||
p.queueId = "main";
|
||||
p.taskIds = {"q2", "q0", "q1"};
|
||||
auto r = dispatcher.on_queue_reorder(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK(r->queue.taskIds.has_value());
|
||||
if (r->queue.taskIds) {
|
||||
const auto& ids = *r->queue.taskIds;
|
||||
CHECK_EQ(ids.size(), std::size_t{3});
|
||||
if (ids.size() == 3) {
|
||||
CHECK_EQ(ids[0], std::string("q2"));
|
||||
CHECK_EQ(ids[1], std::string("q0"));
|
||||
CHECK_EQ(ids[2], std::string("q1"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
proto::QueueReorderParams bad;
|
||||
bad.queueId = "main";
|
||||
bad.taskIds = {"q0", "q1"}; // missing q2: not a permutation
|
||||
auto bad_r = dispatcher.on_queue_reorder(bad);
|
||||
CHECK(!bad_r.has_value());
|
||||
if (!bad_r) CHECK(bad_r.error().code == proto::ErrorCode::InvalidParams);
|
||||
|
||||
proto::QueueReorderParams unknown_queue;
|
||||
unknown_queue.queueId = "does-not-exist";
|
||||
unknown_queue.taskIds = {};
|
||||
auto unknown_r = dispatcher.on_queue_reorder(unknown_queue);
|
||||
CHECK(!unknown_r.has_value());
|
||||
|
||||
tasks.remove("q0");
|
||||
tasks.remove("q1");
|
||||
tasks.remove("q2");
|
||||
}
|
||||
|
||||
// --- schedule.get / schedule.set ---------------------------------------------------
|
||||
{
|
||||
proto::ScheduleGetParams g;
|
||||
g.queueId = std::string("main");
|
||||
auto before = dispatcher.on_schedule_get(g);
|
||||
CHECK(before.has_value());
|
||||
if (before) {
|
||||
CHECK_EQ(before->items.size(), std::size_t{1});
|
||||
if (!before->items.empty()) CHECK(!before->items[0].schedule.has_value());
|
||||
}
|
||||
|
||||
proto::ScheduleSetParams set;
|
||||
set.queueId = "main";
|
||||
proto::Schedule sched;
|
||||
sched.enabled = true;
|
||||
sched.mode = proto::ScheduleMode::Periodic;
|
||||
sched.startTime = std::string("22:00");
|
||||
sched.stopTime = std::string("06:00");
|
||||
set.schedule = sched;
|
||||
auto set_r = dispatcher.on_schedule_set(set);
|
||||
CHECK(set_r.has_value());
|
||||
if (set_r) {
|
||||
CHECK(set_r->schedule.has_value());
|
||||
CHECK(!set_r->nextRunAt.has_value()); // documented gap, not computed
|
||||
}
|
||||
|
||||
auto after = dispatcher.on_schedule_get(g);
|
||||
CHECK(after.has_value());
|
||||
if (after && !after->items.empty())
|
||||
CHECK(after->items[0].schedule.has_value());
|
||||
|
||||
// Clearing (the wire-level "explicit null" is unreachable through the generated
|
||||
// parser's collapsing of absent/null — see UpdatePatch's own comment; this test
|
||||
// only proves set-a-value works, not clear).
|
||||
|
||||
proto::ScheduleGetParams all;
|
||||
auto all_r = dispatcher.on_schedule_get(all);
|
||||
CHECK(all_r.has_value());
|
||||
if (all_r) CHECK(!all_r->items.empty());
|
||||
|
||||
proto::ScheduleGetParams missing;
|
||||
missing.queueId = std::string("does-not-exist");
|
||||
auto missing_r = dispatcher.on_schedule_get(missing);
|
||||
CHECK(missing_r.has_value());
|
||||
if (missing_r) CHECK(missing_r->items.empty());
|
||||
}
|
||||
|
||||
// --- limiter.get / limiter.set -----------------------------------------------------
|
||||
{
|
||||
auto before = dispatcher.on_limiter_get({});
|
||||
CHECK(before.has_value());
|
||||
if (before) {
|
||||
CHECK(!before->enabled);
|
||||
CHECK_EQ(before->globalBps, std::int64_t{0});
|
||||
}
|
||||
|
||||
proto::Limiter set;
|
||||
set.enabled = true;
|
||||
set.globalBps = 2'000'000;
|
||||
auto set_r = dispatcher.on_limiter_set(set);
|
||||
CHECK(set_r.has_value());
|
||||
if (set_r) {
|
||||
CHECK(set_r->enabled);
|
||||
CHECK_EQ(set_r->globalBps, std::int64_t{2'000'000});
|
||||
}
|
||||
|
||||
auto after = dispatcher.on_limiter_get({});
|
||||
CHECK(after.has_value());
|
||||
if (after) {
|
||||
CHECK(after->enabled);
|
||||
CHECK_EQ(after->globalBps, std::int64_t{2'000'000});
|
||||
}
|
||||
}
|
||||
|
||||
// --- download.update: metadata fields, range validation, not-found ---------------
|
||||
{
|
||||
store::Tasks tasks(*db);
|
||||
CHECK(tasks.insert(task("u0")).has_value());
|
||||
|
||||
proto::DownloadUpdateParams p;
|
||||
p.taskId = "u0";
|
||||
p.patch.description = std::string("a note");
|
||||
p.patch.segments = 4;
|
||||
auto r = dispatcher.on_download_update(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK_EQ(r->taskId, std::string("u0"));
|
||||
}
|
||||
auto row = tasks.get("u0");
|
||||
CHECK(row.has_value() && row->has_value());
|
||||
if (row && *row) {
|
||||
CHECK_EQ((*row)->description.value_or(""), std::string("a note"));
|
||||
CHECK_EQ((*row)->req_segments.value_or(-1), std::int64_t{4});
|
||||
}
|
||||
|
||||
proto::DownloadUpdateParams bad_range;
|
||||
bad_range.taskId = "u0";
|
||||
bad_range.patch.segments = 999;
|
||||
auto bad_r = dispatcher.on_download_update(bad_range);
|
||||
CHECK(!bad_r.has_value());
|
||||
if (!bad_r) CHECK(bad_r.error().code == proto::ErrorCode::InvalidParams);
|
||||
|
||||
proto::DownloadUpdateParams missing;
|
||||
missing.taskId = "does-not-exist";
|
||||
missing.patch.description = std::string("x");
|
||||
auto missing_r = dispatcher.on_download_update(missing);
|
||||
CHECK(!missing_r.has_value());
|
||||
if (!missing_r) CHECK(missing_r.error().code == proto::ErrorCode::TaskNotFound);
|
||||
|
||||
tasks.remove("u0");
|
||||
}
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
@@ -0,0 +1,142 @@
|
||||
// settings.get / settings.set: the field <-> SettingKey <-> JSON-type mapping table in
|
||||
// dispatcher.cpp, called directly (no socket) against an in-memory store.
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "store/migrations.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "velox_proto.hpp"
|
||||
|
||||
using namespace velox::daemon;
|
||||
namespace proto = velox::proto;
|
||||
|
||||
void run() {
|
||||
auto db = store::Db::open(":memory:");
|
||||
CHECK(db.has_value());
|
||||
if (!db) return;
|
||||
CHECK(store::migrate_to_head(*db).has_value());
|
||||
|
||||
rpc::EventHub hub;
|
||||
rpc::VeloxDispatcher dispatcher(*db, hub);
|
||||
|
||||
// --- get with keys=nullopt: every one of the 43 SettingKeys comes back -----------
|
||||
{
|
||||
proto::SettingsGetParams p;
|
||||
auto r = dispatcher.on_settings_get(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
int present = 0;
|
||||
present += r->values.general_launchOnLogin.has_value();
|
||||
present += r->values.capture_enabled.has_value();
|
||||
present += r->values.saveTo_allowedRoots.has_value();
|
||||
present += r->values.connection_maxConcurrentDownloads.has_value();
|
||||
present += r->values.proxy_mode.has_value();
|
||||
present += r->values.sounds_onError.has_value();
|
||||
CHECK_EQ(present, 6);
|
||||
// A default, sight-checked: connection.maxConcurrentDownloads is 5 (settings.cpp).
|
||||
CHECK_EQ(r->values.connection_maxConcurrentDownloads.value_or(-1), std::int64_t{5});
|
||||
CHECK(r->values.saveTo_allowedRoots.has_value());
|
||||
if (r->values.saveTo_allowedRoots)
|
||||
CHECK_EQ(r->values.saveTo_allowedRoots->size(), std::size_t{1});
|
||||
// Enum-typed default round-trips through parse_XXX correctly.
|
||||
CHECK(r->values.proxy_mode == proto::SettingsProxyMode::System);
|
||||
}
|
||||
}
|
||||
|
||||
// --- get with an explicit key list: only those fields are populated --------------
|
||||
{
|
||||
proto::SettingsGetParams p;
|
||||
p.keys = {proto::SettingKey::GeneralLaunchOnLogin,
|
||||
proto::SettingKey::ConnectionMaxConcurrentDownloads};
|
||||
auto r = dispatcher.on_settings_get(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK(r->values.general_launchOnLogin.has_value());
|
||||
CHECK(r->values.connection_maxConcurrentDownloads.has_value());
|
||||
CHECK(!r->values.capture_enabled.has_value());
|
||||
CHECK(!r->values.proxy_mode.has_value());
|
||||
}
|
||||
}
|
||||
|
||||
// --- set: a valid change is persisted, reported in `changed`, and readable back ---
|
||||
{
|
||||
proto::SettingsSetParams p;
|
||||
p.values.connection_maxConcurrentDownloads = 12;
|
||||
p.values.general_launchOnLogin = true;
|
||||
auto r = dispatcher.on_settings_set(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK_EQ(r->changed.size(), std::size_t{2});
|
||||
CHECK_EQ(r->values.connection_maxConcurrentDownloads.value_or(-1), std::int64_t{12});
|
||||
CHECK_EQ(r->values.general_launchOnLogin.value_or(false), true);
|
||||
}
|
||||
|
||||
proto::SettingsGetParams g;
|
||||
g.keys = {proto::SettingKey::ConnectionMaxConcurrentDownloads};
|
||||
auto g_r = dispatcher.on_settings_get(g);
|
||||
CHECK(g_r.has_value());
|
||||
if (g_r) CHECK_EQ(g_r->values.connection_maxConcurrentDownloads.value_or(-1), std::int64_t{12});
|
||||
}
|
||||
|
||||
// --- set: setting the same value again reports it unchanged ----------------------
|
||||
{
|
||||
proto::SettingsSetParams p;
|
||||
p.values.connection_maxConcurrentDownloads = 12;
|
||||
auto r = dispatcher.on_settings_set(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) CHECK(r->changed.empty());
|
||||
}
|
||||
|
||||
// --- set: out of range is rejected, and nothing else in the same call lands ------
|
||||
{
|
||||
proto::SettingsSetParams p;
|
||||
p.values.connection_maxConcurrentDownloads = 999; // max 64
|
||||
p.values.general_minimizeToTray = true; // would otherwise succeed
|
||||
auto r = dispatcher.on_settings_set(p);
|
||||
CHECK(!r.has_value());
|
||||
if (!r) CHECK(r.error().code == proto::ErrorCode::InvalidParams);
|
||||
|
||||
proto::SettingsGetParams g;
|
||||
g.keys = {proto::SettingKey::GeneralMinimizeToTray};
|
||||
auto g_r = dispatcher.on_settings_get(g);
|
||||
CHECK(g_r.has_value());
|
||||
// Rejected as a whole: minimizeToTray was never written even though its own value
|
||||
// was valid on its own.
|
||||
if (g_r) CHECK_EQ(g_r->values.general_minimizeToTray.value_or(true), false);
|
||||
}
|
||||
|
||||
// --- set: an enum field round-trips ------------------------------------------------
|
||||
{
|
||||
proto::SettingsSetParams p;
|
||||
p.values.proxy_mode = proto::SettingsProxyMode::Socks5;
|
||||
auto r = dispatcher.on_settings_set(p);
|
||||
CHECK(r.has_value());
|
||||
if (r) {
|
||||
CHECK_EQ(r->changed.size(), std::size_t{1});
|
||||
CHECK(r->values.proxy_mode == proto::SettingsProxyMode::Socks5);
|
||||
}
|
||||
}
|
||||
|
||||
// --- set: saveTo.defaultDir outside every allowed root is rejected with -32011 ----
|
||||
{
|
||||
proto::SettingsSetParams p;
|
||||
p.values.saveTo_defaultDir = "/definitely/not/an/allowed/root";
|
||||
auto r = dispatcher.on_settings_set(p);
|
||||
CHECK(!r.has_value());
|
||||
if (!r) CHECK(r.error().code == proto::ErrorCode::InvalidPath);
|
||||
}
|
||||
|
||||
// --- set: saveTo.allowedRoots with an unresolvable entry is rejected with -32011 --
|
||||
{
|
||||
proto::SettingsSetParams p;
|
||||
p.values.saveTo_allowedRoots = {"/this/path/should/never/exist/anywhere"};
|
||||
auto r = dispatcher.on_settings_set(p);
|
||||
CHECK(!r.has_value());
|
||||
if (!r) CHECK(r.error().code == proto::ErrorCode::InvalidPath);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
@@ -0,0 +1,68 @@
|
||||
#include "rpc/event_hub.hpp"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "check.hpp"
|
||||
|
||||
using velox::daemon::rpc::EventHub;
|
||||
namespace proto = velox::proto;
|
||||
|
||||
void run() {
|
||||
EventHub hub;
|
||||
|
||||
// --- no subscribers: publish is a no-op, not an error --------------------------
|
||||
hub.publish(proto::Event::TaskAdded, nlohmann::json{{"x", 1}});
|
||||
|
||||
// --- a subscription with no filter set receives nothing -------------------
|
||||
std::vector<nlohmann::json> a;
|
||||
const auto sub_a = hub.subscribe([&](const nlohmann::json& n) { a.push_back(n); });
|
||||
hub.publish(proto::Event::TaskAdded, nlohmann::json{{"n", 1}});
|
||||
CHECK_EQ(a.size(), 0u);
|
||||
|
||||
// --- filtering by event kind ------------------------------------------------
|
||||
hub.set_filter(sub_a, {proto::Event::TaskAdded}, std::nullopt);
|
||||
hub.publish(proto::Event::TaskAdded, nlohmann::json{{"n", 2}});
|
||||
hub.publish(proto::Event::TaskRemoved, nlohmann::json{{"n", 3}}); // not subscribed
|
||||
CHECK_EQ(a.size(), 1u);
|
||||
CHECK_EQ(a[0]["n"].get<int>(), 2);
|
||||
|
||||
// --- two subscribers, independent filters -----------------------------------
|
||||
std::vector<nlohmann::json> b;
|
||||
const auto sub_b = hub.subscribe([&](const nlohmann::json& n) { b.push_back(n); });
|
||||
hub.set_filter(sub_b, {proto::Event::TaskRemoved}, std::nullopt);
|
||||
hub.publish(proto::Event::TaskAdded, nlohmann::json{{"n", 4}});
|
||||
hub.publish(proto::Event::TaskRemoved, nlohmann::json{{"n", 5}});
|
||||
CHECK_EQ(a.size(), 2u); // got the TaskAdded
|
||||
CHECK_EQ(b.size(), 1u); // got the TaskRemoved
|
||||
CHECK_EQ(b[0]["n"].get<int>(), 5);
|
||||
|
||||
// --- per-task filter: only the named ids reach the subscriber ---------------
|
||||
std::vector<nlohmann::json> c;
|
||||
const auto sub_c = hub.subscribe([&](const nlohmann::json& n) { c.push_back(n); });
|
||||
hub.set_filter(sub_c, {proto::Event::TaskState},
|
||||
std::vector<std::string>{"t1", "t2"});
|
||||
hub.publish(proto::Event::TaskState, nlohmann::json{{"n", 6}}, "t1");
|
||||
hub.publish(proto::Event::TaskState, nlohmann::json{{"n", 7}}, "t9"); // filtered out
|
||||
hub.publish(proto::Event::TaskState, nlohmann::json{{"n", 8}}, "t2");
|
||||
CHECK_EQ(c.size(), 2u);
|
||||
CHECK_EQ(c[0]["n"].get<int>(), 6);
|
||||
CHECK_EQ(c[1]["n"].get<int>(), 8);
|
||||
|
||||
// --- a non-task-scoped publish (no task_id) reaches a task-filtered sub too --
|
||||
// (matches "session.subscribe narrows task events" — a global event isn't one)
|
||||
hub.publish(proto::Event::TaskState, nlohmann::json{{"n", 9}});
|
||||
CHECK_EQ(c.size(), 3u);
|
||||
|
||||
// --- set_filter replaces, it does not add -----------------------------------
|
||||
hub.set_filter(sub_a, {proto::Event::TaskRemoved}, std::nullopt);
|
||||
hub.publish(proto::Event::TaskAdded, nlohmann::json{{"n", 10}});
|
||||
CHECK_EQ(a.size(), 2u); // still 2 -- TaskAdded no longer reaches sub_a
|
||||
|
||||
// --- unsubscribe stops delivery ------------------------------------------
|
||||
hub.unsubscribe(sub_b);
|
||||
hub.publish(proto::Event::TaskRemoved, nlohmann::json{{"n", 11}});
|
||||
CHECK_EQ(b.size(), 1u); // unchanged
|
||||
CHECK_EQ(a.size(), 3u); // sub_a still gets TaskRemoved
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
@@ -3,7 +3,10 @@
|
||||
|
||||
#include <string>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "sched/fake_engine_port.hpp"
|
||||
#include "sched/governor.hpp"
|
||||
#include "sched/scheduler.hpp"
|
||||
@@ -91,10 +94,10 @@ void run() {
|
||||
CHECK_EQ(engine.starts.size(), 1u); // b0 only
|
||||
CHECK_EQ(task_state(*db, "b0"), std::string("probing"));
|
||||
|
||||
sched.on_engine_state("b0", "downloading", std::nullopt);
|
||||
sched.on_engine_state("b0", "probing", "downloading", std::nullopt);
|
||||
CHECK_EQ(task_state(*db, "b0"), std::string("downloading"));
|
||||
|
||||
sched.on_engine_state("b0", "complete", std::nullopt);
|
||||
sched.on_engine_state("b0", "downloading", "complete", std::nullopt);
|
||||
CHECK_EQ(task_state(*db, "b0"), std::string("complete"));
|
||||
|
||||
CHECK(sched.tick().has_value()); // b0 terminal -> b1 admitted
|
||||
@@ -113,7 +116,7 @@ void run() {
|
||||
CHECK(tasks.insert(task("q0", "queued", "2026-09-10T10:00:00Z", "main", 0)).has_value());
|
||||
CHECK(sched.tick().has_value());
|
||||
CHECK_EQ(engine.starts.size(), 1u);
|
||||
sched.on_engine_state("q0", "downloading", std::nullopt);
|
||||
sched.on_engine_state("q0", "connecting", "downloading", std::nullopt);
|
||||
|
||||
CHECK(db->exec("UPDATE queues SET state='stopped' WHERE queue_id='main'").has_value());
|
||||
CHECK(sched.tick().has_value());
|
||||
@@ -143,7 +146,7 @@ void run() {
|
||||
ef.code = "auth_required";
|
||||
ef.message = "401";
|
||||
ef.http_status = 401;
|
||||
sched.on_engine_state("auth", "paused", ef);
|
||||
sched.on_engine_state("auth", "connecting", "paused", ef);
|
||||
|
||||
store::Tasks t(*db);
|
||||
auto row = t.get("auth").value().value();
|
||||
@@ -186,6 +189,292 @@ void run() {
|
||||
CHECK_EQ(engine.max_active_segments.back(), 12u);
|
||||
CHECK_EQ(sched.reload_config().has_value() ? 0 : 1, 0);
|
||||
}
|
||||
|
||||
// --- event.task.state: published on admission, on a paused transition, and on an
|
||||
// engine-reported transition; previousState reflects the store's prior row ----------
|
||||
{
|
||||
for (const char* id : {"r0", "r1", "r2"}) tasks.remove(id); // leftover from above
|
||||
FakeEnginePort engine;
|
||||
rpc::EventHub hub;
|
||||
Scheduler sched(*db, engine, Governor(GovernorConfig{.max_concurrent_downloads = 10,
|
||||
.max_active_segments = 32}),
|
||||
&hub);
|
||||
std::vector<nlohmann::json> received;
|
||||
const auto sub = hub.subscribe([&](const nlohmann::json& n) { received.push_back(n); });
|
||||
hub.set_filter(sub, {velox::proto::Event::TaskState}, std::nullopt);
|
||||
|
||||
CHECK(tasks.insert(task("ev0", "queued", "2026-09-10T10:00:00Z")).has_value());
|
||||
CHECK(sched.tick().has_value()); // admits ev0: queued -> probing
|
||||
CHECK_EQ(received.size(), 1u);
|
||||
CHECK_EQ(received[0]["params"]["taskId"].get<std::string>(), std::string("ev0"));
|
||||
CHECK_EQ(received[0]["params"]["state"].get<std::string>(), std::string("probing"));
|
||||
CHECK_EQ(received[0]["params"]["previousState"].get<std::string>(), std::string("queued"));
|
||||
CHECK(received[0]["params"]["error"].is_null());
|
||||
CHECK_EQ(received[0]["params"]["summary"]["taskId"].get<std::string>(), std::string("ev0"));
|
||||
|
||||
sched.on_engine_state("ev0", "probing", "downloading", std::nullopt);
|
||||
CHECK_EQ(received.size(), 2u);
|
||||
CHECK_EQ(received[1]["params"]["previousState"].get<std::string>(), std::string("probing"));
|
||||
CHECK_EQ(received[1]["params"]["state"].get<std::string>(), std::string("downloading"));
|
||||
|
||||
sched::TaskErrorFields ef;
|
||||
ef.code = "connection_reset";
|
||||
ef.message = "reset";
|
||||
sched.on_engine_state("ev0", "downloading", "paused", ef);
|
||||
CHECK_EQ(received.size(), 3u);
|
||||
CHECK(!received[2]["params"]["error"].is_null());
|
||||
CHECK_EQ(received[2]["params"]["error"]["code"].get<std::string>(),
|
||||
std::string("connection_reset"));
|
||||
|
||||
tasks.remove("ev0");
|
||||
}
|
||||
|
||||
// --- progress_snapshot: only started tasks, plus a store side-effect --------------
|
||||
{
|
||||
FakeEnginePort engine;
|
||||
Scheduler sched(*db, engine,
|
||||
Governor(GovernorConfig{.max_concurrent_downloads = 10,
|
||||
.max_active_segments = 32}));
|
||||
CHECK(tasks.insert(task("pr0", "queued", "2026-09-10T10:00:00Z")).has_value());
|
||||
CHECK(sched.tick().has_value());
|
||||
CHECK_EQ(engine.starts.size(), 1u);
|
||||
|
||||
vdm::task::Progress p;
|
||||
p.downloaded = 12345;
|
||||
p.speed_bps = 999;
|
||||
p.effective_segments = 4;
|
||||
p.effective_buffer_bytes = 65536;
|
||||
engine.fake_progress[engine.starts[0].id.value] = p;
|
||||
|
||||
const auto snap = sched.progress_snapshot();
|
||||
CHECK_EQ(snap.size(), 1u);
|
||||
CHECK_EQ(snap[0].task_id, std::string("pr0"));
|
||||
CHECK_EQ(snap[0].downloaded_bytes, 12345u);
|
||||
CHECK_EQ(snap[0].speed_bps, 999u);
|
||||
|
||||
auto row = tasks.get("pr0").value().value();
|
||||
CHECK_EQ(row.downloaded_bytes, 12345);
|
||||
CHECK_EQ(row.eff_segments, 4);
|
||||
CHECK((row.eff_buffer_bytes.has_value() && *row.eff_buffer_bytes == 65536));
|
||||
|
||||
tasks.remove("pr0");
|
||||
}
|
||||
|
||||
// --- user_pause / user_resume: a live task, and one that never started -----------
|
||||
{
|
||||
FakeEnginePort engine;
|
||||
Scheduler sched(*db, engine,
|
||||
Governor(GovernorConfig{.max_concurrent_downloads = 10,
|
||||
.max_active_segments = 32}));
|
||||
CHECK(tasks.insert(task("up0", "queued", "2026-09-10T10:00:00Z")).has_value());
|
||||
CHECK(tasks.insert(task("up1", "paused", "2026-09-10T10:00:00Z")).has_value());
|
||||
CHECK(sched.tick().has_value()); // admits up0; up1 stays paused (governor never
|
||||
// touches a user-owned pause)
|
||||
CHECK_EQ(engine.starts.size(), 1u);
|
||||
const auto live_id = engine.starts[0].id;
|
||||
|
||||
// Pausing a live task calls the engine now and transitions eagerly — not left for
|
||||
// the next tick.
|
||||
auto r = sched.user_pause("up0");
|
||||
CHECK(r.found);
|
||||
CHECK(r.changed);
|
||||
CHECK_EQ(r.state, std::string("paused"));
|
||||
CHECK_EQ(engine.paused.size(), 1u);
|
||||
CHECK_EQ(engine.paused[0].value, live_id.value);
|
||||
CHECK_EQ(task_state(*db, "up0"), std::string("paused"));
|
||||
auto row = tasks.get("up0").value().value();
|
||||
CHECK_EQ(row.pause_reason.value_or(""), std::string("user"));
|
||||
|
||||
// Idempotent: pausing an already-paused task is a no-op, not an error.
|
||||
auto again = sched.user_pause("up0");
|
||||
CHECK(again.found);
|
||||
CHECK(!again.changed);
|
||||
|
||||
// Resuming a task that still holds a live engine handle calls engine.resume() and
|
||||
// goes straight to `connecting`.
|
||||
auto res = sched.user_resume("up0");
|
||||
CHECK(res.found);
|
||||
CHECK(res.changed);
|
||||
CHECK_EQ(res.state, std::string("connecting"));
|
||||
CHECK_EQ(engine.resumed.size(), 1u);
|
||||
CHECK_EQ(engine.resumed[0].value, live_id.value);
|
||||
|
||||
// The engine's own delayed pause-ack (on_state with no error, arriving after the
|
||||
// eager transition already wrote the real reason) must not clobber pause_reason
|
||||
// back to NULL.
|
||||
(void)sched.user_pause("up0");
|
||||
sched.on_engine_state("up0", "downloading", "paused", std::nullopt);
|
||||
auto row2 = tasks.get("up0").value().value();
|
||||
CHECK_EQ(row2.pause_reason.value_or(""), std::string("user"));
|
||||
|
||||
// up1 never started (still parked, no engine handle): resume just re-queues it for
|
||||
// the next tick's normal admission.
|
||||
auto res2 = sched.user_resume("up1");
|
||||
CHECK(res2.found);
|
||||
CHECK(res2.changed);
|
||||
CHECK_EQ(res2.state, std::string("queued"));
|
||||
CHECK(engine.resumed.size() == 1u); // up1 was never mapped; no engine call
|
||||
|
||||
// Not found: a bogus id reports found=false, not a crash.
|
||||
auto missing = sched.user_pause("does-not-exist");
|
||||
CHECK(!missing.found);
|
||||
|
||||
tasks.remove("up0");
|
||||
tasks.remove("up1");
|
||||
}
|
||||
|
||||
// --- user_cancel + pause_queue --------------------------------------------------
|
||||
{
|
||||
CHECK(db->exec("UPDATE queues SET state='running' WHERE queue_id='main'").has_value());
|
||||
FakeEnginePort engine;
|
||||
Scheduler sched(*db, engine,
|
||||
Governor(GovernorConfig{.max_concurrent_downloads = 10,
|
||||
.max_active_segments = 32}));
|
||||
CHECK(tasks.insert(task("uc0", "queued", "2026-09-10T10:00:00Z", "main", 0)).has_value());
|
||||
CHECK(tasks.insert(task("uc1", "queued", "2026-09-10T10:00:01Z", "main", 1)).has_value());
|
||||
CHECK(sched.tick().has_value());
|
||||
CHECK_EQ(engine.starts.size(), 2u);
|
||||
|
||||
auto c = sched.user_cancel("uc0", /*discard_partial=*/true);
|
||||
CHECK(c.found);
|
||||
CHECK(c.changed);
|
||||
CHECK_EQ(c.state, std::string("cancelled"));
|
||||
CHECK_EQ(engine.cancelled.size(), 1u);
|
||||
CHECK(engine.cancelled[0].second); // discard_partial passed through
|
||||
CHECK_EQ(task_state(*db, "uc0"), std::string("cancelled"));
|
||||
|
||||
// Cancelling an already-terminal task is a no-op.
|
||||
auto c2 = sched.user_cancel("uc0", false);
|
||||
CHECK(c2.found);
|
||||
CHECK(!c2.changed);
|
||||
|
||||
// pause_queue pauses every still-running task in the queue (uc0 is terminal, so
|
||||
// only uc1 is affected) and reports pause_reason 'queue_stopped'.
|
||||
const auto paused_ids = sched.pause_queue("main");
|
||||
CHECK_EQ(paused_ids.size(), std::size_t{1});
|
||||
CHECK_EQ(paused_ids[0], std::string("uc1"));
|
||||
CHECK_EQ(task_state(*db, "uc1"), std::string("paused"));
|
||||
auto row = tasks.get("uc1").value().value();
|
||||
CHECK_EQ(row.pause_reason.value_or(""), std::string("queue_stopped"));
|
||||
|
||||
tasks.remove("uc0");
|
||||
tasks.remove("uc1");
|
||||
CHECK(db->exec("UPDATE queues SET state='stopped' WHERE queue_id='main'").has_value());
|
||||
}
|
||||
|
||||
// --- probe_now: success (with a category guess) and a mapped failure ------------
|
||||
{
|
||||
FakeEnginePort engine;
|
||||
Scheduler sched(*db, engine,
|
||||
Governor(GovernorConfig{.max_concurrent_downloads = 10,
|
||||
.max_active_segments = 32}));
|
||||
|
||||
vdm::net::ProbeResult pr;
|
||||
pr.effective_url = "https://cdn.example/movie.mp4";
|
||||
pr.filename_from_url = "movie.mp4"; // suggest_filename() needs this set; the real
|
||||
// Prober fills it from the URL path itself
|
||||
pr.total_size = 123456;
|
||||
pr.mime = "video/mp4";
|
||||
pr.resumable = true;
|
||||
pr.accept_ranges = true;
|
||||
pr.etag = "\"abc\"";
|
||||
engine.auto_probe_result = vdm::Result<vdm::net::ProbeResult>{pr};
|
||||
|
||||
velox::proto::DownloadProbeParams params;
|
||||
params.url = "https://cdn.example/movie.mp4";
|
||||
std::optional<velox::proto::HandlerResult<velox::proto::DownloadProbeResult>> got;
|
||||
sched.probe_now(params, [&](auto r) { got = std::move(r); });
|
||||
CHECK(got.has_value());
|
||||
CHECK(got->has_value());
|
||||
if (got && *got) {
|
||||
const auto& r = **got;
|
||||
CHECK_EQ(r.sizeBytes.value_or(-1), std::int64_t{123456});
|
||||
CHECK(r.resumable);
|
||||
CHECK_EQ(r.mime, std::string("video/mp4"));
|
||||
// movie.mp4 -> the 'video' built-in category by extension.
|
||||
CHECK_EQ(r.suggestedCategoryId, std::string("video"));
|
||||
}
|
||||
|
||||
vdm::ErrorInfo err;
|
||||
err.code = vdm::Error::connect_failed;
|
||||
err.context = "connection refused";
|
||||
engine.auto_probe_result = vdm::Result<vdm::net::ProbeResult>{err};
|
||||
std::optional<velox::proto::HandlerResult<velox::proto::DownloadProbeResult>> got2;
|
||||
sched.probe_now(params, [&](auto r) { got2 = std::move(r); });
|
||||
CHECK(got2.has_value());
|
||||
CHECK(!got2->has_value());
|
||||
if (got2 && !*got2)
|
||||
CHECK(got2->error().code == velox::proto::ErrorCode::ProbeFailed);
|
||||
}
|
||||
|
||||
// --- refresh_url: content unchanged, content changed, and a live-handle swap -----
|
||||
{
|
||||
FakeEnginePort engine;
|
||||
Scheduler sched(*db, engine,
|
||||
Governor(GovernorConfig{.max_concurrent_downloads = 10,
|
||||
.max_active_segments = 32}));
|
||||
store::TaskRow r;
|
||||
r.task_id = "ru0";
|
||||
r.url = "https://cdn.example/old-signed-url";
|
||||
r.save_dir = "/tmp";
|
||||
r.filename = "ru0.bin";
|
||||
r.state = "queued";
|
||||
r.created_at = "2026-09-12T00:00:00Z";
|
||||
r.size_bytes = 1000;
|
||||
r.etag = "\"same\"";
|
||||
CHECK(tasks.insert(r).has_value());
|
||||
CHECK(sched.tick().has_value());
|
||||
CHECK_EQ(engine.starts.size(), 1u);
|
||||
const auto live_id = engine.starts[0].id;
|
||||
|
||||
// Same size and etag: not changed.
|
||||
vdm::net::ProbeResult pr;
|
||||
pr.effective_url = "https://cdn.example/new-signed-url";
|
||||
pr.total_size = 1000;
|
||||
pr.etag = "\"same\"";
|
||||
pr.resumable = true;
|
||||
engine.auto_probe_result = vdm::Result<vdm::net::ProbeResult>{pr};
|
||||
|
||||
std::optional<velox::proto::HandlerResult<velox::proto::DownloadRefreshUrlResult>> got;
|
||||
sched.refresh_url("ru0", "https://cdn.example/new-signed-url", std::nullopt, std::nullopt,
|
||||
[&](auto r2) { got = std::move(r2); });
|
||||
CHECK(got.has_value());
|
||||
CHECK(got->has_value());
|
||||
if (got && *got) {
|
||||
CHECK((*got)->ok);
|
||||
CHECK(!(*got)->contentChanged);
|
||||
CHECK((*got)->resumable);
|
||||
}
|
||||
// The live handle got its URL swapped, not a fresh start().
|
||||
CHECK_EQ(engine.starts.size(), 1u);
|
||||
CHECK_EQ(engine.refreshed_urls.size(), std::size_t{1});
|
||||
if (!engine.refreshed_urls.empty()) {
|
||||
CHECK_EQ(engine.refreshed_urls[0].id.value, live_id.value);
|
||||
CHECK_EQ(engine.refreshed_urls[0].url, std::string("https://cdn.example/new-signed-url"));
|
||||
}
|
||||
auto row = tasks.get("ru0").value().value();
|
||||
CHECK_EQ(row.url, std::string("https://cdn.example/new-signed-url"));
|
||||
|
||||
// A different size: content changed.
|
||||
pr.total_size = 2000;
|
||||
engine.auto_probe_result = vdm::Result<vdm::net::ProbeResult>{pr};
|
||||
std::optional<velox::proto::HandlerResult<velox::proto::DownloadRefreshUrlResult>> got2;
|
||||
sched.refresh_url("ru0", "https://cdn.example/another-url", std::nullopt, std::nullopt,
|
||||
[&](auto r2) { got2 = std::move(r2); });
|
||||
CHECK(got2.has_value() && got2->has_value());
|
||||
if (got2 && *got2) CHECK((*got2)->contentChanged);
|
||||
|
||||
// Unknown task: TaskNotFound, no probe issued.
|
||||
const auto probes_before = engine.probe_requests.size();
|
||||
std::optional<velox::proto::HandlerResult<velox::proto::DownloadRefreshUrlResult>> got3;
|
||||
sched.refresh_url("does-not-exist", "https://x/y", std::nullopt, std::nullopt,
|
||||
[&](auto r2) { got3 = std::move(r2); });
|
||||
CHECK(got3.has_value() && !got3->has_value());
|
||||
if (got3 && !*got3) CHECK(got3->error().code == velox::proto::ErrorCode::TaskNotFound);
|
||||
CHECK_EQ(engine.probe_requests.size(), probes_before);
|
||||
|
||||
tasks.remove("ru0");
|
||||
}
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
// The single-instance lock: isolated by runtime dir, not just by euid. This is the bug a
|
||||
// leaked test veloxd exploited — one abstract-socket name per user meant every isolated
|
||||
// instance (real daemon, tests, other lanes) fought over the same lock.
|
||||
|
||||
#include <unistd.h>
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/platform/instance_lock.hpp"
|
||||
|
||||
using namespace velox::daemon::rpc::platform;
|
||||
|
||||
void run() {
|
||||
// Two different runtime dirs: both acquire the lock independently.
|
||||
{
|
||||
const int a = acquire_instance_lock("/run/user/1000/velox-test-a");
|
||||
const int b = acquire_instance_lock("/run/user/1000/velox-test-b");
|
||||
CHECK(a >= 0);
|
||||
CHECK(b >= 0);
|
||||
if (a >= 0) ::close(a);
|
||||
if (b >= 0) ::close(b);
|
||||
}
|
||||
|
||||
// Same runtime dir: the second attempt is refused while the first still holds it.
|
||||
{
|
||||
const int first = acquire_instance_lock("/run/user/1000/velox-test-shared");
|
||||
CHECK(first >= 0);
|
||||
const int second = acquire_instance_lock("/run/user/1000/velox-test-shared");
|
||||
CHECK(second < 0);
|
||||
if (first >= 0) ::close(first);
|
||||
if (second >= 0) ::close(second);
|
||||
|
||||
// Releasing (closing) the fd frees the abstract-namespace name immediately — a
|
||||
// third attempt at the same dir succeeds once the first is gone.
|
||||
const int third = acquire_instance_lock("/run/user/1000/velox-test-shared");
|
||||
CHECK(third >= 0);
|
||||
if (third >= 0) ::close(third);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
@@ -0,0 +1,179 @@
|
||||
// store/categories + store/queues: the two D3 handlers GUI's category panel and queue
|
||||
// view need against a real daemon.
|
||||
|
||||
#include <algorithm>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
#include "check.hpp"
|
||||
#include "store/categories.hpp"
|
||||
#include "store/migrations.hpp"
|
||||
#include "store/queues.hpp"
|
||||
#include "store/sqlite.hpp"
|
||||
#include "store/tasks.hpp"
|
||||
|
||||
using namespace velox::daemon::store;
|
||||
|
||||
void run() {
|
||||
auto db = Db::open(":memory:");
|
||||
CHECK(db.has_value());
|
||||
if (!db) return;
|
||||
CHECK(migrate_to_head(*db).has_value());
|
||||
|
||||
// --- categories: the six seeded built-ins, builtin first --------------------
|
||||
{
|
||||
Categories categories(*db);
|
||||
auto items = categories.list();
|
||||
CHECK(items.has_value());
|
||||
if (items) {
|
||||
CHECK_EQ(items->size(), 6u);
|
||||
for (const auto& c : *items) CHECK(c.builtin);
|
||||
const auto& programs =
|
||||
*std::find_if(items->begin(), items->end(),
|
||||
[](const auto& c) { return c.categoryId == "programs"; });
|
||||
CHECK(!programs.extensions.empty());
|
||||
CHECK(std::find(programs.extensions.begin(), programs.extensions.end(), "iso") !=
|
||||
programs.extensions.end());
|
||||
}
|
||||
}
|
||||
|
||||
// --- queues: the seeded "main" queue, empty task list ------------------------
|
||||
{
|
||||
Queues queues(*db);
|
||||
auto items = queues.list();
|
||||
CHECK(items.has_value());
|
||||
if (items) {
|
||||
CHECK_EQ(items->size(), 1u);
|
||||
CHECK_EQ(items->front().queueId, std::string("main"));
|
||||
CHECK(items->front().taskIds.has_value());
|
||||
CHECK(items->front().taskIds->empty());
|
||||
}
|
||||
}
|
||||
|
||||
// --- queue.taskIds reflects membership, in queue_position order ------------
|
||||
{
|
||||
Tasks tasks(*db);
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
TaskRow r;
|
||||
r.task_id = "t" + std::to_string(i);
|
||||
r.url = "https://example.com/" + r.task_id;
|
||||
r.save_dir = "/tmp";
|
||||
r.filename = r.task_id;
|
||||
r.created_at = "2026-09-11T00:00:00Z";
|
||||
r.queue_id = "main";
|
||||
r.queue_position = 2 - i; // reverse insertion order
|
||||
CHECK(tasks.insert(r).has_value());
|
||||
}
|
||||
Queues queues(*db);
|
||||
auto items = queues.list();
|
||||
CHECK(items.has_value());
|
||||
if (items && !items->empty()) {
|
||||
const auto& ids = *items->front().taskIds;
|
||||
CHECK_EQ(ids.size(), 3u);
|
||||
CHECK_EQ(ids[0], std::string("t2")); // queue_position 0
|
||||
CHECK_EQ(ids[1], std::string("t1"));
|
||||
CHECK_EQ(ids[2], std::string("t0"));
|
||||
}
|
||||
}
|
||||
|
||||
// --- Categories::upsert: create generates an id, builtin is never settable ------
|
||||
{
|
||||
Categories categories(*db);
|
||||
|
||||
velox::proto::Category in;
|
||||
in.name = "ISOs";
|
||||
in.saveDir = "/tmp/isos";
|
||||
in.extensions = {"iso"};
|
||||
in.builtin = true; // ignored on create: a client cannot mint a builtin category
|
||||
auto created = categories.upsert(in);
|
||||
CHECK(created.has_value());
|
||||
if (created) {
|
||||
CHECK(!created->categoryId.empty());
|
||||
CHECK(!created->builtin);
|
||||
|
||||
// A replace keeps builtin=false too, and can rename/re-point.
|
||||
velox::proto::Category patch = *created;
|
||||
patch.name = "ISO Images";
|
||||
patch.builtin = true; // still ignored
|
||||
auto replaced = categories.upsert(patch);
|
||||
CHECK(replaced.has_value());
|
||||
if (replaced) {
|
||||
CHECK_EQ(replaced->categoryId, created->categoryId);
|
||||
CHECK_EQ(replaced->name, std::string("ISO Images"));
|
||||
CHECK(!replaced->builtin);
|
||||
}
|
||||
|
||||
// A builtin category is untouched by remove(), and its tasks are not
|
||||
// reassigned away from it — the store enforces this even without the
|
||||
// dispatcher's own -32602 pre-check.
|
||||
auto builtin_attempt = categories.remove("general", std::nullopt);
|
||||
CHECK(builtin_attempt.has_value());
|
||||
if (builtin_attempt) CHECK(!builtin_attempt->removed);
|
||||
|
||||
// remove() reassigns member tasks (default target: "general") and deletes
|
||||
// the row.
|
||||
Tasks tasks(*db);
|
||||
TaskRow r;
|
||||
r.task_id = "cat-owner";
|
||||
r.url = "https://example.com/x";
|
||||
r.save_dir = "/tmp";
|
||||
r.filename = "x";
|
||||
r.created_at = "2026-09-11T00:00:00Z";
|
||||
r.category_id = created->categoryId;
|
||||
CHECK(tasks.insert(r).has_value());
|
||||
|
||||
auto removed = categories.remove(created->categoryId, std::nullopt);
|
||||
CHECK(removed.has_value());
|
||||
if (removed) {
|
||||
CHECK(removed->removed);
|
||||
CHECK_EQ(removed->reassigned_task_ids.size(), std::size_t{1});
|
||||
CHECK_EQ(removed->reassigned_task_ids[0], std::string("cat-owner"));
|
||||
}
|
||||
auto owner = tasks.get("cat-owner");
|
||||
CHECK(owner.has_value() && owner->has_value());
|
||||
if (owner && *owner)
|
||||
CHECK_EQ((*owner)->category_id.value_or(""), std::string("general"));
|
||||
|
||||
// Gone: a second remove() finds nothing.
|
||||
auto gone = categories.remove(created->categoryId, std::nullopt);
|
||||
CHECK(gone.has_value());
|
||||
if (gone) CHECK(!gone->removed);
|
||||
|
||||
tasks.remove("cat-owner");
|
||||
}
|
||||
}
|
||||
|
||||
// --- Queues::upsert: create generates an id; replace keeps the run state --------
|
||||
{
|
||||
Queues queues(*db);
|
||||
|
||||
velox::proto::Queue in;
|
||||
in.name = "Nightly";
|
||||
in.state = velox::proto::QueueState::Running; // ignored on create: always 'stopped'
|
||||
in.maxConcurrent = 3;
|
||||
auto created = queues.upsert(in);
|
||||
CHECK(created.has_value());
|
||||
if (created) {
|
||||
CHECK(!created->queueId.empty());
|
||||
CHECK(created->state == velox::proto::QueueState::Stopped);
|
||||
|
||||
CHECK(queues.set_state(created->queueId, "running").has_value());
|
||||
|
||||
velox::proto::Queue patch = *created;
|
||||
patch.name = "Nightly Batch";
|
||||
patch.maxConcurrent = 5;
|
||||
patch.state = velox::proto::QueueState::Stopped; // ignored on replace too
|
||||
auto replaced = queues.upsert(patch);
|
||||
CHECK(replaced.has_value());
|
||||
if (replaced) {
|
||||
CHECK_EQ(replaced->name, std::string("Nightly Batch"));
|
||||
CHECK_EQ(replaced->maxConcurrent, std::int64_t{5});
|
||||
// Run state survived the config edit — still 'running' from set_state above,
|
||||
// not reset by the payload's (ignored) 'stopped'.
|
||||
CHECK(replaced->state == velox::proto::QueueState::Running);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
@@ -86,6 +86,56 @@ void run() {
|
||||
.has_value());
|
||||
}
|
||||
|
||||
// --- 0003: start_mode is rebuilt to the contract's values, existing rows mapped -
|
||||
{
|
||||
auto db = Db::open(":memory:");
|
||||
CHECK(db.has_value());
|
||||
if (!db) return;
|
||||
// Build a real pre-0003 db (schema 1..2) with rows in the old, non-contract
|
||||
// start_mode spelling, the way an actually-released daemon would have them.
|
||||
for (const auto& m : embedded_migrations()) {
|
||||
if (m.version > 2) break;
|
||||
CHECK(db->exec(m.sql).has_value());
|
||||
CHECK(db->set_user_version(m.version).has_value());
|
||||
}
|
||||
CHECK(db->exec("INSERT INTO tasks(task_id,url,save_dir,created_at,start_mode) "
|
||||
"VALUES('auto1','http://x','/tmp','2026-09-10T00:00:00Z','auto')")
|
||||
.has_value());
|
||||
CHECK(db->exec("INSERT INTO tasks(task_id,url,save_dir,created_at,start_mode) "
|
||||
"VALUES('man1','http://x','/tmp','2026-09-10T00:00:00Z','manual')")
|
||||
.has_value());
|
||||
CHECK(db->exec("INSERT INTO tasks(task_id,url,save_dir,created_at,start_mode) "
|
||||
"VALUES('q1','http://x','/tmp','2026-09-10T00:00:00Z','queue')")
|
||||
.has_value());
|
||||
|
||||
CHECK(migrate_to_head(*db).has_value());
|
||||
CHECK_EQ(db->user_version(), head);
|
||||
|
||||
auto start_mode_of = [&](const char* id) -> std::string {
|
||||
auto st = db->prepare("SELECT start_mode FROM tasks WHERE task_id=?1");
|
||||
if (!st || !st->bind(1, std::string_view(id))) return "";
|
||||
auto row = st->step();
|
||||
if (!row || !*row) return "";
|
||||
return std::string(st->column_text(0));
|
||||
};
|
||||
CHECK_EQ(start_mode_of("auto1"), std::string("now"));
|
||||
CHECK_EQ(start_mode_of("man1"), std::string("later"));
|
||||
CHECK_EQ(start_mode_of("q1"), std::string("queue")); // passes through unchanged
|
||||
|
||||
// The bug this migration closes: 'later' — a real, documented StartMode value —
|
||||
// used to hit the old CHECK and fail every insert. It's accepted now, and the two
|
||||
// retired spellings are gone for good.
|
||||
CHECK(db->exec("INSERT INTO tasks(task_id,url,save_dir,created_at,start_mode) "
|
||||
"VALUES('later1','http://x','/tmp','2026-09-10T00:00:00Z','later')")
|
||||
.has_value());
|
||||
CHECK(db->exec("INSERT INTO tasks(task_id,url,save_dir,created_at,start_mode) "
|
||||
"VALUES('bad1','http://x','/tmp','2026-09-10T00:00:00Z','auto')")
|
||||
.has_value() == false);
|
||||
CHECK(db->exec("INSERT INTO tasks(task_id,url,save_dir,created_at,start_mode) "
|
||||
"VALUES('bad2','http://x','/tmp','2026-09-10T00:00:00Z','manual')")
|
||||
.has_value() == false);
|
||||
}
|
||||
|
||||
// --- re-running the migrator on an at-head DB is a no-op ------------------------
|
||||
{
|
||||
auto db = Db::open(":memory:");
|
||||
@@ -100,11 +150,20 @@ void run() {
|
||||
}
|
||||
|
||||
// --- forward-only: from every released version [0 .. head-1], reach head --------
|
||||
// A "released version N" db has the real schema migrations 1..N actually built, not
|
||||
// just the pragma set to N — faking the pragma alone left `start >= 1` cases running
|
||||
// a later migration (e.g. 0002's ALTER TABLE tasks / rebuild of segments) against a
|
||||
// db with no tables at all.
|
||||
for (std::int64_t start = 0; start < head; ++start) {
|
||||
auto db = Db::open(":memory:");
|
||||
CHECK(db.has_value());
|
||||
if (!db) continue;
|
||||
CHECK(db->set_user_version(start).has_value());
|
||||
for (const auto& m : embedded_migrations()) {
|
||||
if (m.version > start) break;
|
||||
CHECK(db->exec(m.sql).has_value());
|
||||
CHECK(db->set_user_version(m.version).has_value());
|
||||
}
|
||||
CHECK_EQ(db->user_version(), start);
|
||||
auto out = migrate_to_head(*db);
|
||||
CHECK(out.has_value());
|
||||
if (out) {
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
// systemd_activated_fd(): the LISTEN_PID/LISTEN_FDS contract, without a real systemd.
|
||||
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cstdlib>
|
||||
#include <string>
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/systemd_activation.hpp"
|
||||
|
||||
using namespace velox::daemon::rpc;
|
||||
|
||||
namespace {
|
||||
void set_env(const char* k, const std::string& v) { ::setenv(k, v.c_str(), 1); }
|
||||
} // namespace
|
||||
|
||||
void run() {
|
||||
// Not activated: neither var set.
|
||||
::unsetenv("LISTEN_PID");
|
||||
::unsetenv("LISTEN_FDS");
|
||||
CHECK_EQ(systemd_activated_fd(), -1);
|
||||
|
||||
// LISTEN_PID for a different process: not us, so not activated.
|
||||
set_env("LISTEN_PID", std::to_string(::getpid() + 1));
|
||||
set_env("LISTEN_FDS", "1");
|
||||
CHECK_EQ(systemd_activated_fd(), -1);
|
||||
// Consumed regardless of the outcome — a stale value from some other process's
|
||||
// exec chain must not leak into what veloxd checks next time.
|
||||
CHECK(::getenv("LISTEN_PID") == nullptr);
|
||||
CHECK(::getenv("LISTEN_FDS") == nullptr);
|
||||
|
||||
// Our own pid, LISTEN_FDS=1: activated, fd 3 (SD_LISTEN_FDS_START).
|
||||
set_env("LISTEN_PID", std::to_string(::getpid()));
|
||||
set_env("LISTEN_FDS", "1");
|
||||
CHECK_EQ(systemd_activated_fd(), 3);
|
||||
CHECK(::getenv("LISTEN_PID") == nullptr);
|
||||
|
||||
// Our own pid but LISTEN_FDS=2: a unit file mismatch (veloxd only ever asks for one
|
||||
// socket) — refuse rather than guess which of two fds is the right one.
|
||||
set_env("LISTEN_PID", std::to_string(::getpid()));
|
||||
set_env("LISTEN_FDS", "2");
|
||||
CHECK_EQ(systemd_activated_fd(), -1);
|
||||
|
||||
// Garbage LISTEN_FDS: not activated, not a crash.
|
||||
set_env("LISTEN_PID", std::to_string(::getpid()));
|
||||
set_env("LISTEN_FDS", "not-a-number");
|
||||
CHECK_EQ(systemd_activated_fd(), -1);
|
||||
|
||||
::unsetenv("LISTEN_PID");
|
||||
::unsetenv("LISTEN_FDS");
|
||||
}
|
||||
|
||||
TEST_MAIN()
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/event_loop.hpp"
|
||||
#include "rpc/ndjson.hpp"
|
||||
#include "rpc/uds_server.hpp"
|
||||
@@ -71,8 +72,9 @@ void run() {
|
||||
CHECK(velox::daemon::store::migrate_to_head(*db).has_value());
|
||||
|
||||
rpc::EventLoop loop;
|
||||
rpc::VeloxDispatcher dispatcher(*db);
|
||||
rpc::UdsServer server(loop, dispatcher, sock);
|
||||
rpc::EventHub hub;
|
||||
rpc::VeloxDispatcher dispatcher(*db, hub);
|
||||
rpc::UdsServer server(loop, dispatcher, hub, sock);
|
||||
const auto ec = server.start();
|
||||
CHECK(!ec);
|
||||
if (ec) return;
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
#include "check.hpp"
|
||||
#include "rpc/dispatcher.hpp"
|
||||
#include "rpc/event_hub.hpp"
|
||||
#include "rpc/event_loop.hpp"
|
||||
#include "rpc/pairing.hpp"
|
||||
#include "rpc/runtime_dir.hpp"
|
||||
@@ -123,9 +124,10 @@ void run() {
|
||||
rpc::RuntimeDir rt{dir ? dir : "/tmp"};
|
||||
|
||||
rpc::EventLoop loop;
|
||||
rpc::VeloxDispatcher dispatcher(*db);
|
||||
rpc::EventHub hub;
|
||||
rpc::VeloxDispatcher dispatcher(*db, hub);
|
||||
rpc::EnvAutoApprover approver;
|
||||
rpc::WsServer server(loop, dispatcher, *db, approver, rt);
|
||||
rpc::WsServer server(loop, dispatcher, *db, approver, hub, rt);
|
||||
const auto ec = server.start();
|
||||
CHECK(!ec);
|
||||
if (ec) return;
|
||||
|
||||
Vendored
+11
@@ -0,0 +1,11 @@
|
||||
velox (0.1.0-1) UNRELEASED; urgency=medium
|
||||
|
||||
* Local test build only. Not signed, not uploaded anywhere — see
|
||||
packaging/README.md for what this package is and is not.
|
||||
* libveloxcore is linked statically; no libveloxcore.so is shipped.
|
||||
* Placeholder app icon (packaging/icons/) — replace before any real release.
|
||||
* No real pairing-approval UI is built yet (needs libdbus-1-dev, not in this
|
||||
build's Build-Depends): pairing requires VELOX_PAIR_AUTO=1. See
|
||||
packaging/README.md and postinst's own notice.
|
||||
|
||||
-- Velox Local Test Build <[email protected]> Sat, 12 Sep 2026 22:30:43 +0400
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user