Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fdacf732fa | ||
|
|
fb77c9008a | ||
|
|
4f30fa6970 | ||
|
|
bd1bc029f3 | ||
|
|
e442c99130 | ||
|
|
67b7b75336 | ||
|
|
60363a7142 | ||
|
|
ecedfac903 |
+6
-3
@@ -5,9 +5,12 @@ Nobody else commits here. Everybody else *generates from* here.
|
||||
|
||||
> ## Status: **v1.0.0 — FROZEN** (2026-09-09)
|
||||
>
|
||||
> The surface below is complete and generated from: 38 methods, 9 events, 26 named types,
|
||||
> 59 fixtures. See `docs/adr/0005-protocol-1.0.0-freeze.md` for the versioning rule and
|
||||
> `docs/adr/0010-...` for the failure taxonomy and the segment range convention.
|
||||
> **v1.0.0** froze 38 methods, 9 events, 26 named types. **v1.1.0** (current) is a minor
|
||||
> bump on top of it: `bufferBytes` bounds widened to 64 KiB - 16 MiB across all four
|
||||
> locations, two new settings keys (`connection.maxTotalBufferBytes`,
|
||||
> `connection.maxActiveSegments`), and `TaskDetail.effectiveBufferBytes` — see
|
||||
> `docs/adr/0012-buffer-and-segment-budget.md`. See also `docs/adr/0005-...` for the
|
||||
> versioning rule and `docs/adr/0010-...` for the failure taxonomy and segment ranges.
|
||||
>
|
||||
> Lane requests are answered in writing: `contracts/proto-answers-m1.md` responds to
|
||||
> `core/docs/proto-requests-m1.md` point by point.
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
1.0.0
|
||||
1.1.0
|
||||
|
||||
@@ -116,7 +116,8 @@
|
||||
"referrer": "https://releases.ubuntu.com/26.04/",
|
||||
"userAgent": "Velox/0.1",
|
||||
"mime": "application/octet-stream",
|
||||
"bufferBytes": 4194304,
|
||||
"bufferBytes": 16777216,
|
||||
"effectiveBufferBytes": 4194304,
|
||||
"partPath": "/home/sami/Downloads/Programs/ubuntu-26.04-desktop-amd64.iso.veloxpart",
|
||||
"checksum": null,
|
||||
"checksumVerified": null,
|
||||
@@ -128,6 +129,7 @@
|
||||
"segment ranges are contiguous and cover exactly [0, sizeBytes) with no gaps or overlaps",
|
||||
"startByte and endByte are both INCLUSIVE: segment 0 here covers 778567680 bytes, 0 through 778567679, and is copied verbatim into 'Range: bytes=0-778567679'",
|
||||
"segmentDetail has exactly summary.segments entries",
|
||||
"the GUI draws one bar per entry and is never told what a segment steal is"
|
||||
"the GUI draws one bar per entry and is never told what a segment steal is",
|
||||
"bufferBytes is what was requested (16 MiB); effectiveBufferBytes (4 MiB) is what this segment is actually using right now, after connection.maxTotalBufferBytes (128 MiB default) is divided across every live segment in the daemon -- not just this task's -- up to connection.maxActiveSegments (32 default). The clamp is global: a task can be reduced even when its own segment count alone would not force it."
|
||||
]
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@
|
||||
"keys": [
|
||||
"connection.maxSegmentsPerDownload",
|
||||
"connection.bufferBytes",
|
||||
"connection.maxTotalBufferBytes",
|
||||
"connection.maxActiveSegments",
|
||||
"connection.maxConcurrentDownloads",
|
||||
"connection.timeoutSec"
|
||||
]
|
||||
@@ -20,7 +22,9 @@
|
||||
"result": {
|
||||
"values": {
|
||||
"connection.maxSegmentsPerDownload": 8,
|
||||
"connection.bufferBytes": 4194304,
|
||||
"connection.bufferBytes": 1048576,
|
||||
"connection.maxTotalBufferBytes": 134217728,
|
||||
"connection.maxActiveSegments": 32,
|
||||
"connection.maxConcurrentDownloads": 5,
|
||||
"connection.timeoutSec": 30
|
||||
}
|
||||
@@ -29,6 +33,8 @@
|
||||
"assertions": [
|
||||
"only the requested keys come back",
|
||||
"keys null returns everything",
|
||||
"no password is ever present: credentials live in the Secret Service"
|
||||
"no password is ever present: credentials live in the Secret Service",
|
||||
"connection.bufferBytes defaults to 1 MiB (1048576), not the old 4 MiB",
|
||||
"connection.maxTotalBufferBytes and connection.maxActiveSegments are the two knobs behind TaskDetail.effectiveBufferBytes; Options cannot show or set the clamp without them"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -8,7 +8,8 @@
|
||||
"params": {
|
||||
"values": {
|
||||
"connection.maxSegmentsPerDownload": 16,
|
||||
"downloads.verifyChecksums": true
|
||||
"downloads.verifyChecksums": true,
|
||||
"connection.bufferBytes": 2097152
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -18,17 +19,20 @@
|
||||
"result": {
|
||||
"values": {
|
||||
"connection.maxSegmentsPerDownload": 16,
|
||||
"downloads.verifyChecksums": true
|
||||
"downloads.verifyChecksums": true,
|
||||
"connection.bufferBytes": 2097152
|
||||
},
|
||||
"changed": [
|
||||
"connection.maxSegmentsPerDownload",
|
||||
"downloads.verifyChecksums"
|
||||
"downloads.verifyChecksums",
|
||||
"connection.bufferBytes"
|
||||
]
|
||||
}
|
||||
},
|
||||
"assertions": [
|
||||
"event.settings.changed is emitted carrying exactly the keys in changed[]",
|
||||
"an unknown key is -32602 and nothing at all is written",
|
||||
"a directory key naming an unwritable path is -32011"
|
||||
"a directory key naming an unwritable path is -32011",
|
||||
"connection.bufferBytes accepts 64 KiB - 16 MiB; a value outside that range is -32602"
|
||||
]
|
||||
}
|
||||
|
||||
+41
-12
@@ -2,7 +2,7 @@
|
||||
"openrpc": "1.2.6",
|
||||
"info": {
|
||||
"title": "Velox Download Manager",
|
||||
"version": "1.0.0",
|
||||
"version": "1.1.0",
|
||||
"description": "The wire contract between veloxd and every client: the Qt GUI, the CLI, the native-messaging host and the Firefox extension. One JSON-RPC 2.0 payload set over four framings; only the framing differs.\n\nGENERATED from contracts/schema/ by contracts/codegen/gen_openrpc.py. Do not edit by hand.",
|
||||
"license": {
|
||||
"name": "See repository LICENSE"
|
||||
@@ -498,9 +498,11 @@
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"minimum": 4096,
|
||||
"maximum": 8388608
|
||||
}
|
||||
"minimum": 65536,
|
||||
"maximum": 16777216,
|
||||
"description": "Requested write buffer per segment, in bytes. null means use connection.bufferBytes. Default 1 MiB; range 64 KiB - 16 MiB. Silently reduced to fit connection.maxTotalBufferBytes across all live segments; the effective value is reported back as TaskDetail.effectiveBufferBytes."
|
||||
},
|
||||
"description": "Requested write buffer per segment, in bytes. null means use connection.bufferBytes. Default 1 MiB; range 64 KiB - 16 MiB. Silently reduced to fit connection.maxTotalBufferBytes across all live segments; the effective value is reported back as TaskDetail.effectiveBufferBytes."
|
||||
},
|
||||
{
|
||||
"name": "startMode",
|
||||
@@ -1366,8 +1368,9 @@
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"minimum": 4096,
|
||||
"maximum": 8388608
|
||||
"minimum": 65536,
|
||||
"maximum": 16777216,
|
||||
"description": "The REQUESTED write buffer per segment. Subject to the same maxTotalBufferBytes reduction as DownloadSpec.bufferBytes; the effective value comes back on the next download.get."
|
||||
},
|
||||
"checksum": {
|
||||
"oneOf": [
|
||||
@@ -3138,8 +3141,9 @@
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"minimum": 4096,
|
||||
"maximum": 8388608
|
||||
"minimum": 65536,
|
||||
"maximum": 16777216,
|
||||
"description": "Requested write buffer per segment, in bytes. null means use connection.bufferBytes. Default 1 MiB; range 64 KiB - 16 MiB. Silently reduced to fit connection.maxTotalBufferBytes across all live segments; the effective value is reported back as TaskDetail.effectiveBufferBytes."
|
||||
},
|
||||
"startMode": {
|
||||
"$ref": "#/components/schemas/StartMode"
|
||||
@@ -3760,6 +3764,8 @@
|
||||
"connection.preset",
|
||||
"connection.maxSegmentsPerDownload",
|
||||
"connection.bufferBytes",
|
||||
"connection.maxTotalBufferBytes",
|
||||
"connection.maxActiveSegments",
|
||||
"connection.maxConcurrentDownloads",
|
||||
"connection.timeoutSec",
|
||||
"connection.maxRetries",
|
||||
@@ -3883,8 +3889,9 @@
|
||||
},
|
||||
"connection.bufferBytes": {
|
||||
"type": "integer",
|
||||
"minimum": 4096,
|
||||
"maximum": 8388608
|
||||
"minimum": 65536,
|
||||
"maximum": 16777216,
|
||||
"description": "Default per-segment write buffer, in bytes, when a task does not request its own. Default 1 MiB (1048576); range 64 KiB - 16 MiB. This is the single biggest throughput knob and is exposed in Options -> Downloads -> 'Write buffer per connection'."
|
||||
},
|
||||
"connection.maxConcurrentDownloads": {
|
||||
"type": "integer",
|
||||
@@ -3973,6 +3980,18 @@
|
||||
},
|
||||
"sounds.onError": {
|
||||
"type": "string"
|
||||
},
|
||||
"connection.maxTotalBufferBytes": {
|
||||
"type": "integer",
|
||||
"minimum": 16777216,
|
||||
"maximum": 2147483648,
|
||||
"description": "Global cap on write-buffer memory across every live segment, in bytes. Default 128 MiB (134217728). Every live segment's buffer is reduced to fit maxTotalBufferBytes / (live segment count, capped at maxActiveSegments); the reduced value is reported per task as TaskDetail.effectiveBufferBytes. Exists so a burst of large downloads with a large per-segment buffer cannot exhaust memory."
|
||||
},
|
||||
"connection.maxActiveSegments": {
|
||||
"type": "integer",
|
||||
"minimum": 1,
|
||||
"maximum": 256,
|
||||
"description": "Global ceiling on segments actually transferring at once, across every task. Default 32. This is the real bound behind '20 active downloads': the rest of each download's segments queue rather than all dialling out simultaneously. DAEMON's scheduler needs this value to decide what to admit; CORE enforces it."
|
||||
}
|
||||
},
|
||||
"title": "Settings"
|
||||
@@ -4040,8 +4059,18 @@
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"minimum": 4096,
|
||||
"maximum": 8388608
|
||||
"minimum": 65536,
|
||||
"maximum": 16777216,
|
||||
"description": "The REQUESTED write buffer per segment. See effectiveBufferBytes for what is actually in use."
|
||||
},
|
||||
"effectiveBufferBytes": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"minimum": 65536,
|
||||
"maximum": 16777216,
|
||||
"description": "The write buffer actually in use per live segment, right now. May be well below bufferBytes: the daemon reduces every live segment's buffer to fit connection.maxTotalBufferBytes across connection.maxActiveSegments concurrently-transferring segments, and reports the reduced value here so the GUI can show '16 MiB (using 4 MiB)'. null before the task has started its first segment."
|
||||
},
|
||||
"partPath": {
|
||||
"type": [
|
||||
|
||||
@@ -99,7 +99,7 @@ Agreed with your ranking: these are minor under rule 4 and land as small PRs to
|
||||
|
||||
| # | Item | Verdict | Shape |
|
||||
|---|---|---|---|
|
||||
| **B2a** | readable effective buffer size | **accepted** | `effectiveBufferBytes` on `TaskSummary`, next to the effective segment count, so the requested/effective split reads the same way for both. You are right about the `additionalProperties: false` trap — no daemon can tack it on, so it needs a schema PR either way. |
|
||||
| **B2a** | readable effective buffer size | **landed in 1.1.0** | `TaskDetail.effectiveBufferBytes` (placed on `TaskDetail`, not `TaskSummary` — `bufferBytes` itself was already `TaskDetail`-only, so the pair stays together). See `docs/adr/0012-buffer-and-segment-budget.md`, which also lands B4's bounds and the two new settings keys in the same PR. |
|
||||
| **F2** | credential return path for 401/407 | **accepted as proposed** | `download.provideAuth {taskId, username, password, save?}` → `{ok}`. Unix socket only, privileged: a credential-bearing method must never be reachable from the browser. Secrets go to the Secret Service; `save` only tells DAEMON whether to persist. |
|
||||
| **F1** | "needs user decision" carrier | **the simple option** | `state: paused` + `event.notify` is the intended carrier for M1: CORE reports `server_file_changed`, DAEMON pauses and notifies, GUI offers restart. A dedicated `event.task.decision` + `download.decide` is a real design with a state machine attached, and it should not be invented in a hurry — raise it again in M3 if the notify path proves too thin. A string comparison on `error.code` covers the engine side either way, which is now a `TaskErrorCode` comparison rather than a magic number. |
|
||||
| **F3** | `checksum` string format | **already frozen, differently** | `download.add {checksum}` is **not** a string. It is a `Checksum` object: `{algorithm: "md5"\|"sha1"\|"sha256"\|"sha512", value: "<hex>"}`, with `value` patterned `^[0-9a-fA-F]{32,128}$`. Parse your `"<algo>:<hex>"` form at the CLI or GUI edge, not on the wire. Note `sha512` is accepted by the contract even though the appendix lists MD5/SHA-256 — reject it in the engine if you do not implement it, rather than the contract forbidding it. |
|
||||
|
||||
@@ -78,8 +78,9 @@
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"minimum": 4096,
|
||||
"maximum": 8388608
|
||||
"minimum": 65536,
|
||||
"maximum": 16777216,
|
||||
"description": "The REQUESTED write buffer per segment. Subject to the same maxTotalBufferBytes reduction as DownloadSpec.bufferBytes; the effective value comes back on the next download.get."
|
||||
},
|
||||
"checksum": {
|
||||
"oneOf": [
|
||||
|
||||
@@ -87,8 +87,9 @@
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"minimum": 4096,
|
||||
"maximum": 8388608
|
||||
"minimum": 65536,
|
||||
"maximum": 16777216,
|
||||
"description": "Requested write buffer per segment, in bytes. null means use connection.bufferBytes. Default 1 MiB; range 64 KiB - 16 MiB. Silently reduced to fit connection.maxTotalBufferBytes across all live segments; the effective value is reported back as TaskDetail.effectiveBufferBytes."
|
||||
},
|
||||
"startMode": {
|
||||
"$ref": "https://velox.dev/schema/types/StartMode.schema.json"
|
||||
|
||||
@@ -5,25 +5,48 @@
|
||||
"description": "Every settings key that exists. The Options dialog maps 1:1 onto this list and the GUI must not invent a key that is not here. Kept in lockstep with Settings.schema.json by a conformance check.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"general.launchOnLogin", "general.minimizeToTray", "general.showDropTarget",
|
||||
"general.confirmOnExit", "general.language", "general.checkForUpdates",
|
||||
|
||||
"capture.enabled", "capture.monitoredExtensions", "capture.monitoredMimeTypes",
|
||||
"capture.minSizeBytes", "capture.excludedHosts", "capture.bypassModifier",
|
||||
"general.launchOnLogin",
|
||||
"general.minimizeToTray",
|
||||
"general.showDropTarget",
|
||||
"general.confirmOnExit",
|
||||
"general.language",
|
||||
"general.checkForUpdates",
|
||||
"capture.enabled",
|
||||
"capture.monitoredExtensions",
|
||||
"capture.monitoredMimeTypes",
|
||||
"capture.minSizeBytes",
|
||||
"capture.excludedHosts",
|
||||
"capture.bypassModifier",
|
||||
"capture.autoStartTypes",
|
||||
|
||||
"saveTo.defaultDir", "saveTo.tempDir", "saveTo.allowedRoots",
|
||||
"saveTo.fileExistsPolicy", "saveTo.createSubfolderPerSite",
|
||||
|
||||
"connection.preset", "connection.maxSegmentsPerDownload", "connection.bufferBytes",
|
||||
"connection.maxConcurrentDownloads", "connection.timeoutSec", "connection.maxRetries",
|
||||
"saveTo.defaultDir",
|
||||
"saveTo.tempDir",
|
||||
"saveTo.allowedRoots",
|
||||
"saveTo.fileExistsPolicy",
|
||||
"saveTo.createSubfolderPerSite",
|
||||
"connection.preset",
|
||||
"connection.maxSegmentsPerDownload",
|
||||
"connection.bufferBytes",
|
||||
"connection.maxTotalBufferBytes",
|
||||
"connection.maxActiveSegments",
|
||||
"connection.maxConcurrentDownloads",
|
||||
"connection.timeoutSec",
|
||||
"connection.maxRetries",
|
||||
"connection.retryBackoffSec",
|
||||
|
||||
"downloads.speedLimitBps", "downloads.speedLimitEnabled", "downloads.virusScanCommand",
|
||||
"downloads.postDownloadCommand", "downloads.duplicatePolicy", "downloads.verifyChecksums",
|
||||
|
||||
"proxy.mode", "proxy.host", "proxy.port", "proxy.username", "proxy.bypassHosts", "proxy.pacUrl",
|
||||
|
||||
"sounds.enabled", "sounds.onComplete", "sounds.onQueueComplete", "sounds.onError"
|
||||
"downloads.speedLimitBps",
|
||||
"downloads.speedLimitEnabled",
|
||||
"downloads.virusScanCommand",
|
||||
"downloads.postDownloadCommand",
|
||||
"downloads.duplicatePolicy",
|
||||
"downloads.verifyChecksums",
|
||||
"proxy.mode",
|
||||
"proxy.host",
|
||||
"proxy.port",
|
||||
"proxy.username",
|
||||
"proxy.bypassHosts",
|
||||
"proxy.pacUrl",
|
||||
"sounds.enabled",
|
||||
"sounds.onComplete",
|
||||
"sounds.onQueueComplete",
|
||||
"sounds.onError"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -101,8 +101,9 @@
|
||||
},
|
||||
"connection.bufferBytes": {
|
||||
"type": "integer",
|
||||
"minimum": 4096,
|
||||
"maximum": 8388608
|
||||
"minimum": 65536,
|
||||
"maximum": 16777216,
|
||||
"description": "Default per-segment write buffer, in bytes, when a task does not request its own. Default 1 MiB (1048576); range 64 KiB - 16 MiB. This is the single biggest throughput knob and is exposed in Options -> Downloads -> 'Write buffer per connection'."
|
||||
},
|
||||
"connection.maxConcurrentDownloads": {
|
||||
"type": "integer",
|
||||
@@ -191,6 +192,18 @@
|
||||
},
|
||||
"sounds.onError": {
|
||||
"type": "string"
|
||||
},
|
||||
"connection.maxTotalBufferBytes": {
|
||||
"type": "integer",
|
||||
"minimum": 16777216,
|
||||
"maximum": 2147483648,
|
||||
"description": "Global cap on write-buffer memory across every live segment, in bytes. Default 128 MiB (134217728). Every live segment's buffer is reduced to fit maxTotalBufferBytes / (live segment count, capped at maxActiveSegments); the reduced value is reported per task as TaskDetail.effectiveBufferBytes. Exists so a burst of large downloads with a large per-segment buffer cannot exhaust memory."
|
||||
},
|
||||
"connection.maxActiveSegments": {
|
||||
"type": "integer",
|
||||
"minimum": 1,
|
||||
"maximum": 256,
|
||||
"description": "Global ceiling on segments actually transferring at once, across every task. Default 32. This is the real bound behind '20 active downloads': the rest of each download's segments queue rather than all dialling out simultaneously. DAEMON's scheduler needs this value to decide what to admit; CORE enforces it."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,8 +54,18 @@
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"minimum": 4096,
|
||||
"maximum": 8388608
|
||||
"minimum": 65536,
|
||||
"maximum": 16777216,
|
||||
"description": "The REQUESTED write buffer per segment. See effectiveBufferBytes for what is actually in use."
|
||||
},
|
||||
"effectiveBufferBytes": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"minimum": 65536,
|
||||
"maximum": 16777216,
|
||||
"description": "The write buffer actually in use per live segment, right now. May be well below bufferBytes: the daemon reduces every live segment's buffer to fit connection.maxTotalBufferBytes across connection.maxActiveSegments concurrently-transferring segments, and reports the reduced value here so the GUI can show '16 MiB (using 4 MiB)'. null before the task has started its first segment."
|
||||
},
|
||||
"partPath": {
|
||||
"type": [
|
||||
|
||||
+8
-4
@@ -5,16 +5,20 @@
|
||||
# `add_subdirectory(core)` in the root CMakeLists.txt (see core/docs/pkg-requests-m1.md).
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
find_package(CURL 8.0 REQUIRED)
|
||||
|
||||
add_library(veloxcore STATIC
|
||||
src/util/error.cpp
|
||||
src/util/log.cpp
|
||||
src/util/thread_pool.cpp
|
||||
src/net/curl_error.cpp
|
||||
src/net/http_client.cpp
|
||||
)
|
||||
add_library(velox::core ALIAS veloxcore)
|
||||
|
||||
target_include_directories(veloxcore PUBLIC
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/include
|
||||
target_include_directories(veloxcore
|
||||
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
|
||||
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src # net/*.cpp -> "net/curl_error.hpp"
|
||||
)
|
||||
|
||||
target_compile_features(veloxcore PUBLIC cxx_std_23)
|
||||
@@ -25,9 +29,9 @@ target_compile_options(veloxcore PRIVATE
|
||||
-Wall -Wextra -Wpedantic -Werror
|
||||
)
|
||||
|
||||
target_link_libraries(veloxcore PUBLIC Threads::Threads)
|
||||
target_link_libraries(veloxcore PUBLIC Threads::Threads CURL::libcurl)
|
||||
|
||||
# Later stages add: find_package(CURL 8.0) for net/, find_package(OpenSSL) for meta/.
|
||||
# Later stages add: find_package(OpenSSL) for meta/ (streaming SHA-256 + resume CRC).
|
||||
|
||||
if(VELOX_BUILD_TESTS)
|
||||
add_subdirectory(tests)
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
# CORE response to ADR 0011 (admission control & the segment budget)
|
||||
|
||||
**Verdict: accept the decision, with three amendments and one caveat on §3.5.** None of
|
||||
the amendments change DAEMON's task-unit model — `daemon/src/sched/` is unblocked. The
|
||||
caveat tightens what "admission implies progress" can promise.
|
||||
|
||||
Signing off on §1 (each ceiling enforced once, by unit owner), §2 (the single clamp),
|
||||
§4 (per-host caps split by unit from one table), §6 (contract gap / daemon-local stopgap),
|
||||
and the three rejected alternatives — the shared-semaphore rejection especially.
|
||||
|
||||
---
|
||||
|
||||
## Answers to the five open questions
|
||||
|
||||
### Q1 — Min-1 before seconds: implementable in the stealer without inversion? **Yes.**
|
||||
|
||||
Slot allocation is a pure function of `(running tasks, their held-slot counts, their
|
||||
effective per-task caps, DAEMON's priority order, total budget)`. It runs to completion on
|
||||
every budget-changing edge (segment complete / start / pause / fail, task admit / pause /
|
||||
resume, a steal, a cap change), in two ordered passes:
|
||||
|
||||
1. **Guarantee pass.** Walk tasks holding **zero** slots *in DAEMON's priority order*; give
|
||||
each one slot while the pool is non-empty.
|
||||
2. **Growth pass.** While the pool is non-empty, round-robin over running tasks in priority
|
||||
order, granting one slot to any task below its effective cap
|
||||
(`min(spec.segments ?? maxSegmentsPerDownload, host_cap, resumable ? ∞ : 1)`), until the
|
||||
pool empties or nobody wants more.
|
||||
|
||||
The inversion DAEMON is worried about at slot release does not occur, because **a released
|
||||
slot always goes to the pool and the allocator re-runs from pass 1** — it is never handed
|
||||
back locally to the releasing task. If task A's segment finishes and both A and a
|
||||
lower-priority C now hold zero, the guarantee pass processes the zero-slot set *in priority
|
||||
order*, so A gets it back, not C. A task that drops to zero re-enters the guarantee queue
|
||||
at its priority position, not the back.
|
||||
|
||||
Liveness of min-1 depends on DAEMON honouring §2's clamp — never running more tasks than
|
||||
`maxActiveSegments`. If it admits 3 running tasks against a budget of 2, one starves by
|
||||
construction and no fairness rule fixes it. §2 already says this; calling it out because
|
||||
it is load-bearing for Q1.
|
||||
|
||||
### Q2 — Probe pool outside the segment budget, size 4: **confirmed.**
|
||||
|
||||
CORE's probe path is a dedicated pool, independent of `maxActiveSegments`. A probe is a
|
||||
`Request` with `method = HEAD` (or `range = {0,0}`) whose `on_head` returns `abort`; it
|
||||
never allocates a transfer slot. Default pool size 4, exposed as
|
||||
`set_probe_pool_size(uint32_t)`. Probe cancellation is immediate (a timed-out probe frees
|
||||
its pool slot at once), so `capture.offer` answering `ignore`-first-probe-after works.
|
||||
Probes still open real sockets — DAEMON should still bound probe *submission* on its side;
|
||||
CORE bounds *concurrency*, not queue depth.
|
||||
|
||||
### Q3 — `set_max_active_segments()` live-apply: **drain, never kill.**
|
||||
|
||||
- **Raise:** new slots available immediately; allocator runs; starved then growing tasks
|
||||
take them.
|
||||
- **Lower:** in-flight segments run to their next boundary. No new segment starts while
|
||||
`active > new_ceiling`. Completed segments' slots are withheld until
|
||||
`active ≤ new_ceiling`. Nothing is aborted, no partial range is lost.
|
||||
- **Edge:** if `new_ceiling < running_task_count`, CORE honours min-1 for the top
|
||||
`new_ceiling` tasks in priority order; the remainder are held at zero slots and reported
|
||||
in `tasks_starved`. CORE does **not** auto-pause them — that's policy. After a live
|
||||
lower, DAEMON must reconcile its running set against the new clamp (pause the
|
||||
lowest-priority excess).
|
||||
|
||||
### Q4 — Priority shape: **an ordered list, pushed on change — not an integer, not per tick.**
|
||||
|
||||
`set_task_order(std::span<const TaskId>)`, called by DAEMON whenever the order changes
|
||||
(admit, remove, reorder, priority edit, queue switch). CORE caches it and the allocator
|
||||
walks it. An integer priority would force CORE to implement tie-breaking (FIFO by
|
||||
admission time, queue precedence) — that's DAEMON policy and DAEMON already has the total
|
||||
order. Not per tick: an unchanged order isn't re-sent. A running task absent from the list
|
||||
sorts last (shouldn't happen; defensive).
|
||||
|
||||
### Q5 — Budget-change callback coalescing: **4 Hz for counts, immediate on the edge.**
|
||||
|
||||
`on_budget_changed` coalesced at ≤4 Hz to match `event.task.progress` — DAEMON isn't
|
||||
making sub-250 ms admission decisions. **Exception:** fire immediately when
|
||||
`tasks_starved` crosses 0→non-zero or non-zero→0, so DAEMON's invariant check and any UI
|
||||
reaction see that transition without up to 250 ms of lag.
|
||||
|
||||
---
|
||||
|
||||
## Amendments requested to the ADR
|
||||
|
||||
### A1 — §3.3 wording: distinguish *steal* from *yield*
|
||||
|
||||
"A steal is slot-neutral" is true for the steal the ADR means (a worker that finished its
|
||||
range takes the tail of the largest remaining range — the same worker, the same slot).
|
||||
Min-1 also needs a second, non-neutral operation:
|
||||
|
||||
- **Yield** — the allocator marks an over-quota task to release one slot *at its next
|
||||
segment boundary*. When that segment completes the slot goes to the pool → guarantee
|
||||
pass → starved task. It is a slot transfer, not slot-neutral, and it is bounded by the
|
||||
yielding segment's remaining bytes (never a mid-segment kill).
|
||||
|
||||
Please add "yield" to §3.3 as the mechanism that satisfies §3.1 when the budget is full.
|
||||
"Steal" stays exactly as written.
|
||||
|
||||
### A2 — §3.5 / §3.6: "admission implies progress" is bounded-delay, not immediate
|
||||
|
||||
When the budget is full of healthy incumbents, a newly admitted task's first slot
|
||||
materialises only when some incumbent segment reaches a boundary (yield) — bounded by that
|
||||
segment's remaining bytes, hard-capped by the stall timeout (`low_speed_secs`, default
|
||||
30 s, after which a stalled segment fails and frees its slot). So the true bound is
|
||||
|
||||
```
|
||||
time_to_first_slot ≤ min(incumbent's next boundary, low_speed_secs) + connect_timeout
|
||||
```
|
||||
|
||||
not "connect timeout + per-host cap" alone. Two consequences:
|
||||
|
||||
- §3.6's **2 s** assertion window is too tight — a legitimately full budget with a slow
|
||||
incumbent tail can hold a new task at zero for longer than 2 s with nothing wrong.
|
||||
Recommend the warning threshold be `low_speed_secs + connect_timeout_ms` (~45 s), or
|
||||
configurable.
|
||||
- CORE will expose `starved_since` (a monotonic timestamp) per starved task via the
|
||||
diagnostics call below, so DAEMON can tell "briefly waiting for a boundary" from
|
||||
"wedged" without guessing.
|
||||
|
||||
Optional future tightening (not M1): a **preemptive split** — truncate an incumbent's
|
||||
largest remaining range ahead of its current offset and hand the freed tail to the starved
|
||||
task as a new segment. Zero bytes lost, first slot within one round-trip. CORE will add
|
||||
this if the yield delay proves painful in the soak test; it doesn't change the API.
|
||||
|
||||
### A3 — C1 API: add a starved-set accessor and pin two definitions
|
||||
|
||||
```
|
||||
struct EngineBudget { uint32_t total; uint32_t active; uint32_t tasks_starved; };
|
||||
EngineBudget budget() const;
|
||||
uint32_t segments_active(TaskId) const; // slots held, any state
|
||||
std::vector<TaskId> starved_tasks() const; // diagnostics / velox ls --json
|
||||
std::optional<SteadyTime> starved_since(TaskId) const; // per A2
|
||||
void on_budget_changed(std::function<void(EngineBudget)>);
|
||||
```
|
||||
|
||||
Definitions, so the projection to `TaskSummary.segments` (ADR 0010: effective count) is
|
||||
unambiguous:
|
||||
|
||||
- **`segments_active(id)`** = slots the task holds, counting a segment in `connecting`
|
||||
(0 bytes yet) as well as `downloading`. This is what the user sees as "using N
|
||||
connections."
|
||||
- **`tasks_starved`** counts running tasks with `segments_active == 0`. A task with a
|
||||
`connecting` segment is **not** starved — it is progressing.
|
||||
|
||||
---
|
||||
|
||||
## C2–C6 confirmations
|
||||
|
||||
- **C2:** drain-not-kill, per Q3. Confirmed as DAEMON assumed.
|
||||
- **C3:** `set_host_segment_cap(std::string host, uint32_t)` — confirmed. CORE keeps the
|
||||
`host → cap` map, applies it in the per-task effective cap and in the stealer (no Nth
|
||||
connection to a host capped at N-1). DAEMON owns the table; CORE derives a task's host
|
||||
from its URL + mirror set.
|
||||
- **C4:** agreed it's PROTO's, same bundle as CORE's B2a / `buffer-sizing.md` asks
|
||||
(`connection.maxActiveSegments`, `connection.maxTotalBufferBytes`). DAEMON's local-value
|
||||
stopgap is fine.
|
||||
- **C5:** signed off — see Q1 plus amendments A1/A2. Min-1 is buildable; "implies
|
||||
progress" is bounded-delay; steal stays slot-neutral, yield is the transfer op.
|
||||
- **C6:** confirmed — see Q2.
|
||||
|
||||
---
|
||||
|
||||
## New API surface CORE will expose for `sched/` (summary)
|
||||
|
||||
```
|
||||
void set_max_active_segments(uint32_t); // drain-not-kill
|
||||
void set_host_segment_cap(std::string host, uint32_t);
|
||||
void set_task_order(std::span<const TaskId>); // pushed on change
|
||||
void set_probe_pool_size(uint32_t); // default 4
|
||||
EngineBudget budget() const;
|
||||
uint32_t segments_active(TaskId) const;
|
||||
std::vector<TaskId> starved_tasks() const;
|
||||
std::optional<SteadyTime> starved_since(TaskId) const;
|
||||
void on_budget_changed(std::function<void(EngineBudget)>); // 4 Hz + starved edge
|
||||
```
|
||||
|
||||
All of it lands with stage 6 (segmenter/stealer) / stage 8 (download_task). None of it is
|
||||
on the M1 critical path ahead of where DAEMON needs it; flag if the ordering is wrong.
|
||||
@@ -3,7 +3,7 @@
|
||||
//
|
||||
// Source: contracts/schema/**
|
||||
// Generator: contracts/codegen/gen_cpp.py
|
||||
// Contract: v1.0.0
|
||||
// Contract: v1.1.0
|
||||
//
|
||||
// Hand-editing this file is a merge blocker. Fix the schema and regenerate:
|
||||
// python3 contracts/codegen/gen_cpp.py
|
||||
@@ -401,6 +401,8 @@ std::string_view to_string(SettingKey v) noexcept {
|
||||
case SettingKey::ConnectionPreset: return "connection.preset";
|
||||
case SettingKey::ConnectionMaxSegmentsPerDownload: return "connection.maxSegmentsPerDownload";
|
||||
case SettingKey::ConnectionBufferBytes: return "connection.bufferBytes";
|
||||
case SettingKey::ConnectionMaxTotalBufferBytes: return "connection.maxTotalBufferBytes";
|
||||
case SettingKey::ConnectionMaxActiveSegments: return "connection.maxActiveSegments";
|
||||
case SettingKey::ConnectionMaxConcurrentDownloads: return "connection.maxConcurrentDownloads";
|
||||
case SettingKey::ConnectionTimeoutSec: return "connection.timeoutSec";
|
||||
case SettingKey::ConnectionMaxRetries: return "connection.maxRetries";
|
||||
@@ -447,6 +449,8 @@ Result<SettingKey> parse_SettingKey(std::string_view s) {
|
||||
if (s == "connection.preset") return SettingKey::ConnectionPreset;
|
||||
if (s == "connection.maxSegmentsPerDownload") return SettingKey::ConnectionMaxSegmentsPerDownload;
|
||||
if (s == "connection.bufferBytes") return SettingKey::ConnectionBufferBytes;
|
||||
if (s == "connection.maxTotalBufferBytes") return SettingKey::ConnectionMaxTotalBufferBytes;
|
||||
if (s == "connection.maxActiveSegments") return SettingKey::ConnectionMaxActiveSegments;
|
||||
if (s == "connection.maxConcurrentDownloads") return SettingKey::ConnectionMaxConcurrentDownloads;
|
||||
if (s == "connection.timeoutSec") return SettingKey::ConnectionTimeoutSec;
|
||||
if (s == "connection.maxRetries") return SettingKey::ConnectionMaxRetries;
|
||||
@@ -1601,8 +1605,8 @@ template <> Result<DownloadSpec> parse<DownloadSpec>(const nlohmann::json& j, st
|
||||
if (it != j.end() && !it->is_null()) {
|
||||
if (!(*it).is_number_integer()) return std::unexpected(ParseError{std::string(fp), "expected an integer"});
|
||||
auto val = (*it).get<std::int64_t>();
|
||||
if (val < 4096) return std::unexpected(ParseError{std::string(fp), "value is below the minimum of 4096"});
|
||||
if (val > 8388608) return std::unexpected(ParseError{std::string(fp), "value is above the maximum of 8388608"});
|
||||
if (val < 65536) return std::unexpected(ParseError{std::string(fp), "value is below the minimum of 65536"});
|
||||
if (val > 16777216) return std::unexpected(ParseError{std::string(fp), "value is above the maximum of 16777216"});
|
||||
out.bufferBytes = std::move(val);
|
||||
}
|
||||
}
|
||||
@@ -2428,6 +2432,8 @@ void to_json(nlohmann::json& j, const Settings& v) {
|
||||
if (v.sounds_onComplete.has_value()) j["sounds.onComplete"] = *v.sounds_onComplete;
|
||||
if (v.sounds_onQueueComplete.has_value()) j["sounds.onQueueComplete"] = *v.sounds_onQueueComplete;
|
||||
if (v.sounds_onError.has_value()) j["sounds.onError"] = *v.sounds_onError;
|
||||
if (v.connection_maxTotalBufferBytes.has_value()) j["connection.maxTotalBufferBytes"] = *v.connection_maxTotalBufferBytes;
|
||||
if (v.connection_maxActiveSegments.has_value()) j["connection.maxActiveSegments"] = *v.connection_maxActiveSegments;
|
||||
}
|
||||
|
||||
template <> Result<Settings> parse<Settings>(const nlohmann::json& j, std::string_view path) {
|
||||
@@ -2660,8 +2666,8 @@ template <> Result<Settings> parse<Settings>(const nlohmann::json& j, std::strin
|
||||
if (it != j.end() && !it->is_null()) {
|
||||
if (!(*it).is_number_integer()) return std::unexpected(ParseError{std::string(fp), "expected an integer"});
|
||||
auto val = (*it).get<std::int64_t>();
|
||||
if (val < 4096) return std::unexpected(ParseError{std::string(fp), "value is below the minimum of 4096"});
|
||||
if (val > 8388608) return std::unexpected(ParseError{std::string(fp), "value is above the maximum of 8388608"});
|
||||
if (val < 65536) return std::unexpected(ParseError{std::string(fp), "value is below the minimum of 65536"});
|
||||
if (val > 16777216) return std::unexpected(ParseError{std::string(fp), "value is above the maximum of 16777216"});
|
||||
out.connection_bufferBytes = std::move(val);
|
||||
}
|
||||
}
|
||||
@@ -2865,6 +2871,28 @@ template <> Result<Settings> parse<Settings>(const nlohmann::json& j, std::strin
|
||||
out.sounds_onError = std::move(val);
|
||||
}
|
||||
}
|
||||
{
|
||||
const std::string fp = join(path, "connection.maxTotalBufferBytes");
|
||||
const auto it = j.find("connection.maxTotalBufferBytes");
|
||||
if (it != j.end() && !it->is_null()) {
|
||||
if (!(*it).is_number_integer()) return std::unexpected(ParseError{std::string(fp), "expected an integer"});
|
||||
auto val = (*it).get<std::int64_t>();
|
||||
if (val < 16777216) return std::unexpected(ParseError{std::string(fp), "value is below the minimum of 16777216"});
|
||||
if (val > 2147483648) return std::unexpected(ParseError{std::string(fp), "value is above the maximum of 2147483648"});
|
||||
out.connection_maxTotalBufferBytes = std::move(val);
|
||||
}
|
||||
}
|
||||
{
|
||||
const std::string fp = join(path, "connection.maxActiveSegments");
|
||||
const auto it = j.find("connection.maxActiveSegments");
|
||||
if (it != j.end() && !it->is_null()) {
|
||||
if (!(*it).is_number_integer()) return std::unexpected(ParseError{std::string(fp), "expected an integer"});
|
||||
auto val = (*it).get<std::int64_t>();
|
||||
if (val < 1) return std::unexpected(ParseError{std::string(fp), "value is below the minimum of 1"});
|
||||
if (val > 256) return std::unexpected(ParseError{std::string(fp), "value is above the maximum of 256"});
|
||||
out.connection_maxActiveSegments = std::move(val);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -3183,6 +3211,7 @@ void to_json(nlohmann::json& j, const TaskDetail& v) {
|
||||
if (v.userAgent.has_value()) j["userAgent"] = *v.userAgent;
|
||||
if (v.mime.has_value()) j["mime"] = *v.mime;
|
||||
if (v.bufferBytes.has_value()) j["bufferBytes"] = *v.bufferBytes;
|
||||
if (v.effectiveBufferBytes.has_value()) j["effectiveBufferBytes"] = *v.effectiveBufferBytes;
|
||||
if (v.partPath.has_value()) j["partPath"] = *v.partPath;
|
||||
if (v.checksum.has_value()) j["checksum"] = *v.checksum;
|
||||
if (v.checksumVerified.has_value()) j["checksumVerified"] = *v.checksumVerified;
|
||||
@@ -3264,11 +3293,22 @@ template <> Result<TaskDetail> parse<TaskDetail>(const nlohmann::json& j, std::s
|
||||
if (it != j.end() && !it->is_null()) {
|
||||
if (!(*it).is_number_integer()) return std::unexpected(ParseError{std::string(fp), "expected an integer"});
|
||||
auto val = (*it).get<std::int64_t>();
|
||||
if (val < 4096) return std::unexpected(ParseError{std::string(fp), "value is below the minimum of 4096"});
|
||||
if (val > 8388608) return std::unexpected(ParseError{std::string(fp), "value is above the maximum of 8388608"});
|
||||
if (val < 65536) return std::unexpected(ParseError{std::string(fp), "value is below the minimum of 65536"});
|
||||
if (val > 16777216) return std::unexpected(ParseError{std::string(fp), "value is above the maximum of 16777216"});
|
||||
out.bufferBytes = std::move(val);
|
||||
}
|
||||
}
|
||||
{
|
||||
const std::string fp = join(path, "effectiveBufferBytes");
|
||||
const auto it = j.find("effectiveBufferBytes");
|
||||
if (it != j.end() && !it->is_null()) {
|
||||
if (!(*it).is_number_integer()) return std::unexpected(ParseError{std::string(fp), "expected an integer"});
|
||||
auto val = (*it).get<std::int64_t>();
|
||||
if (val < 65536) return std::unexpected(ParseError{std::string(fp), "value is below the minimum of 65536"});
|
||||
if (val > 16777216) return std::unexpected(ParseError{std::string(fp), "value is above the maximum of 16777216"});
|
||||
out.effectiveBufferBytes = std::move(val);
|
||||
}
|
||||
}
|
||||
{
|
||||
const std::string fp = join(path, "partPath");
|
||||
const auto it = j.find("partPath");
|
||||
@@ -4711,8 +4751,8 @@ template <> Result<DownloadUpdateParamsPatch> parse<DownloadUpdateParamsPatch>(c
|
||||
if (it != j.end() && !it->is_null()) {
|
||||
if (!(*it).is_number_integer()) return std::unexpected(ParseError{std::string(fp), "expected an integer"});
|
||||
auto val = (*it).get<std::int64_t>();
|
||||
if (val < 4096) return std::unexpected(ParseError{std::string(fp), "value is below the minimum of 4096"});
|
||||
if (val > 8388608) return std::unexpected(ParseError{std::string(fp), "value is above the maximum of 8388608"});
|
||||
if (val < 65536) return std::unexpected(ParseError{std::string(fp), "value is below the minimum of 65536"});
|
||||
if (val > 16777216) return std::unexpected(ParseError{std::string(fp), "value is above the maximum of 16777216"});
|
||||
out.bufferBytes = std::move(val);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//
|
||||
// Source: contracts/schema/**
|
||||
// Generator: contracts/codegen/gen_cpp.py
|
||||
// Contract: v1.0.0
|
||||
// Contract: v1.1.0
|
||||
//
|
||||
// Hand-editing this file is a merge blocker. Fix the schema and regenerate:
|
||||
// python3 contracts/codegen/gen_cpp.py
|
||||
@@ -27,7 +27,7 @@
|
||||
// docs/adr/0009-generated-protocol-library.md.
|
||||
namespace velox::proto {
|
||||
|
||||
inline constexpr std::string_view kProtocolVersion = "1.0.0";
|
||||
inline constexpr std::string_view kProtocolVersion = "1.1.0";
|
||||
|
||||
/// Why a payload could not be turned into a typed value. `path` is a JSON Pointer
|
||||
/// into the offending document, so a conformance failure names the exact field.
|
||||
@@ -219,6 +219,8 @@ enum class SettingKey {
|
||||
ConnectionPreset, // "connection.preset"
|
||||
ConnectionMaxSegmentsPerDownload, // "connection.maxSegmentsPerDownload"
|
||||
ConnectionBufferBytes, // "connection.bufferBytes"
|
||||
ConnectionMaxTotalBufferBytes, // "connection.maxTotalBufferBytes"
|
||||
ConnectionMaxActiveSegments, // "connection.maxActiveSegments"
|
||||
ConnectionMaxConcurrentDownloads, // "connection.maxConcurrentDownloads"
|
||||
ConnectionTimeoutSec, // "connection.timeoutSec"
|
||||
ConnectionMaxRetries, // "connection.maxRetries"
|
||||
@@ -535,6 +537,9 @@ struct DownloadSpec {
|
||||
/// per-host cap, and to 1 when the source turns out not to be resumable. What is actually in
|
||||
/// use comes back as TaskSummary.segments. null means use connection.maxSegmentsPerDownload.
|
||||
std::optional<std::int64_t> segments{};
|
||||
/// Requested write buffer per segment, in bytes. null means use connection.bufferBytes. Default
|
||||
/// 1 MiB; range 64 KiB - 16 MiB. Silently reduced to fit connection.maxTotalBufferBytes across
|
||||
/// all live segments; the effective value is reported back as TaskDetail.effectiveBufferBytes.
|
||||
std::optional<std::int64_t> bufferBytes{};
|
||||
std::optional<StartMode> startMode{};
|
||||
std::optional<std::string> description{};
|
||||
@@ -708,6 +713,9 @@ struct Settings {
|
||||
std::optional<bool> saveTo_createSubfolderPerSite{};
|
||||
std::optional<SettingsConnectionPreset> connection_preset{};
|
||||
std::optional<std::int64_t> connection_maxSegmentsPerDownload{};
|
||||
/// Default per-segment write buffer, in bytes, when a task does not request its own. Default 1
|
||||
/// MiB (1048576); range 64 KiB - 16 MiB. This is the single biggest throughput knob and is
|
||||
/// exposed in Options -> Downloads -> 'Write buffer per connection'.
|
||||
std::optional<std::int64_t> connection_bufferBytes{};
|
||||
std::optional<std::int64_t> connection_maxConcurrentDownloads{};
|
||||
std::optional<std::int64_t> connection_timeoutSec{};
|
||||
@@ -729,6 +737,17 @@ struct Settings {
|
||||
std::optional<std::string> sounds_onComplete{};
|
||||
std::optional<std::string> sounds_onQueueComplete{};
|
||||
std::optional<std::string> sounds_onError{};
|
||||
/// Global cap on write-buffer memory across every live segment, in bytes. Default 128 MiB
|
||||
/// (134217728). Every live segment's buffer is reduced to fit maxTotalBufferBytes / (live
|
||||
/// segment count, capped at maxActiveSegments); the reduced value is reported per task as
|
||||
/// TaskDetail.effectiveBufferBytes. Exists so a burst of large downloads with a large
|
||||
/// per-segment buffer cannot exhaust memory.
|
||||
std::optional<std::int64_t> connection_maxTotalBufferBytes{};
|
||||
/// Global ceiling on segments actually transferring at once, across every task. Default 32.
|
||||
/// This is the real bound behind '20 active downloads': the rest of each download's segments
|
||||
/// queue rather than all dialling out simultaneously. DAEMON's scheduler needs this value to
|
||||
/// decide what to admit; CORE enforces it.
|
||||
std::optional<std::int64_t> connection_maxActiveSegments{};
|
||||
};
|
||||
|
||||
/// Why a task is in the failed or retry_wait state. Distinct from the JSON-RPC Error, which
|
||||
@@ -805,7 +824,15 @@ struct TaskDetail {
|
||||
std::optional<std::string> referrer{};
|
||||
std::optional<std::string> userAgent{};
|
||||
std::optional<std::string> mime{};
|
||||
/// The REQUESTED write buffer per segment. See effectiveBufferBytes for what is actually in
|
||||
/// use.
|
||||
std::optional<std::int64_t> bufferBytes{};
|
||||
/// The write buffer actually in use per live segment, right now. May be well below bufferBytes:
|
||||
/// the daemon reduces every live segment's buffer to fit connection.maxTotalBufferBytes across
|
||||
/// connection.maxActiveSegments concurrently-transferring segments, and reports the reduced
|
||||
/// value here so the GUI can show '16 MiB (using 4 MiB)'. null before the task has started its
|
||||
/// first segment.
|
||||
std::optional<std::int64_t> effectiveBufferBytes{};
|
||||
/// Absolute path of the .veloxpart file while the task is unfinished.
|
||||
std::optional<std::string> partPath{};
|
||||
std::optional<Checksum> checksum{};
|
||||
@@ -1030,6 +1057,8 @@ struct DownloadUpdateParamsPatch {
|
||||
/// as DownloadSpec.segments. Takes effect on the next start; a running task is not re-segmented
|
||||
/// underneath the user.
|
||||
std::optional<std::int64_t> segments{};
|
||||
/// The REQUESTED write buffer per segment. Subject to the same maxTotalBufferBytes reduction as
|
||||
/// DownloadSpec.bufferBytes; the effective value comes back on the next download.get.
|
||||
std::optional<std::int64_t> bufferBytes{};
|
||||
std::optional<Checksum> checksum{};
|
||||
};
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
// vdm/net/http_client.hpp — a libcurl-multi wrapper for the download engine.
|
||||
//
|
||||
// One HttpClient owns a small pool of worker threads, each with its own curl_multi
|
||||
// (curl handles are not thread-safe; a handle lives on exactly one worker for its life).
|
||||
// The engine hands it a Request plus callbacks and gets back a Transfer handle it can
|
||||
// pause / resume / cancel from any thread.
|
||||
//
|
||||
// Layering: this is the ONLY core header that pulls in libcurl, and only in its .cpp —
|
||||
// nothing here exposes a curl type. No JSON / SQL / Qt / RPC (CLAUDE.md §3).
|
||||
//
|
||||
// Callbacks run ON THE WORKER THREAD, one transfer at a time for a given Transfer.
|
||||
// They must not block (that stalls every other transfer on that worker) and must not
|
||||
// call back into this Transfer's pause/resume/cancel re-entrantly — post that work
|
||||
// elsewhere. on_data must not allocate on the hot path (AGENT-CORE); the ring buffer it
|
||||
// writes into is preallocated by the caller (stage 4).
|
||||
//
|
||||
// This header compiles standalone.
|
||||
|
||||
#ifndef VDM_NET_HTTP_CLIENT_HPP
|
||||
#define VDM_NET_HTTP_CLIENT_HPP
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "vdm/net/http_types.hpp"
|
||||
#include "vdm/util/bytes.hpp"
|
||||
#include "vdm/util/result.hpp"
|
||||
|
||||
namespace vdm::net {
|
||||
|
||||
// What on_data tells the client to do with the transfer after this chunk.
|
||||
enum class DataAction {
|
||||
proceed, // keep receiving
|
||||
pause, // stop receiving; resumes on Transfer::resume()
|
||||
abort, // end the transfer now (finishes with Error::canceled)
|
||||
};
|
||||
|
||||
struct TransferStats {
|
||||
std::uint64_t bytes_received = 0;
|
||||
long http_status = 0;
|
||||
std::string effective_url;
|
||||
// Timings in milliseconds (from CURLINFO_*_TIME_T), 0 if the phase didn't happen.
|
||||
long namelookup_ms = 0;
|
||||
long connect_ms = 0;
|
||||
long appconnect_ms = 0; // TLS handshake done
|
||||
long starttransfer_ms = 0; // first response byte
|
||||
long total_ms = 0;
|
||||
};
|
||||
|
||||
struct TransferCallbacks {
|
||||
// Response headers are in. Called at most once per transfer (a followed redirect's
|
||||
// intermediate headers are not delivered). If the caller only wanted headers (a
|
||||
// probe), return DataAction::abort here.
|
||||
std::function<DataAction(const ResponseHead &)> on_head;
|
||||
|
||||
// A chunk of body bytes. The span is valid only for the duration of the call.
|
||||
std::function<DataAction(ConstByteSpan)> on_data;
|
||||
|
||||
// The transfer ended — success carries stats, failure carries the mapped error
|
||||
// (ErrorInfo.http_status is set for HTTP-status failures). Always called exactly once,
|
||||
// last.
|
||||
std::function<void(Result<TransferStats>)> on_finished;
|
||||
};
|
||||
|
||||
class HttpClient;
|
||||
|
||||
// Lightweight handle to a running transfer. Copyable (shared state). All methods are
|
||||
// safe to call from any thread; they post to the owning worker and return immediately.
|
||||
// Dropping the last handle does NOT cancel — call cancel() for that.
|
||||
class Transfer {
|
||||
public:
|
||||
Transfer() = default;
|
||||
|
||||
[[nodiscard]] std::uint64_t id() const noexcept;
|
||||
[[nodiscard]] bool valid() const noexcept { return static_cast<bool>(state_); }
|
||||
|
||||
void pause(); // no-op if already paused / finished
|
||||
void resume(); // no-op if not paused / finished
|
||||
void cancel(); // idempotent; on_finished fires with Error::canceled
|
||||
|
||||
private:
|
||||
friend class HttpClient;
|
||||
struct State;
|
||||
explicit Transfer(std::shared_ptr<State> s) : state_(std::move(s)) {}
|
||||
std::shared_ptr<State> state_;
|
||||
};
|
||||
|
||||
class HttpClient {
|
||||
public:
|
||||
struct Options {
|
||||
// Worker threads, each with its own curl_multi. New transfers are assigned
|
||||
// round-robin. 0 => pick from hardware_concurrency (min 1, max 4).
|
||||
unsigned workers = 0;
|
||||
// CURLMOPT_MAX_TOTAL_CONNECTIONS per worker (0 = curl default).
|
||||
long max_connections_per_worker = 0;
|
||||
// Shared DNS + TLS-session cache across this client's workers (curl_share).
|
||||
bool share_dns_and_tls = true;
|
||||
};
|
||||
|
||||
HttpClient(); // default Options
|
||||
explicit HttpClient(Options opts);
|
||||
~HttpClient();
|
||||
|
||||
HttpClient(const HttpClient &) = delete;
|
||||
HttpClient &operator=(const HttpClient &) = delete;
|
||||
|
||||
[[nodiscard]] unsigned worker_count() const noexcept;
|
||||
|
||||
// Start a transfer. The Request is consumed (moved). Returns an invalid Transfer and
|
||||
// never calls the callbacks only if the client is shutting down; every other failure
|
||||
// (bad URL, DNS, ...) is delivered through on_finished.
|
||||
Transfer start(Request req, TransferCallbacks cbs);
|
||||
|
||||
private:
|
||||
friend class Transfer; // Transfer posts pause/resume/cancel commands to Impl
|
||||
struct Impl;
|
||||
std::unique_ptr<Impl> impl_;
|
||||
};
|
||||
|
||||
} // namespace vdm::net
|
||||
|
||||
#endif // VDM_NET_HTTP_CLIENT_HPP
|
||||
@@ -0,0 +1,156 @@
|
||||
// vdm/net/http_types.hpp — value types for HTTP requests and responses.
|
||||
//
|
||||
// No libcurl in this header: it is the vocabulary the rest of core/ speaks to the net
|
||||
// layer. http_client.hpp is the only place curl leaks in, and only in its .cpp.
|
||||
//
|
||||
// This header compiles standalone.
|
||||
|
||||
#ifndef VDM_NET_HTTP_TYPES_HPP
|
||||
#define VDM_NET_HTTP_TYPES_HPP
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace vdm::net {
|
||||
|
||||
enum class Method { get, head };
|
||||
|
||||
[[nodiscard]] constexpr std::string_view method_name(Method m) noexcept {
|
||||
return m == Method::head ? "HEAD" : "GET";
|
||||
}
|
||||
|
||||
// A closed byte range [first, last]. `last == kUnbounded` means "to the end of the
|
||||
// resource" (`Range: bytes=first-`). This mirrors HTTP Range semantics — inclusive on
|
||||
// both ends — deliberately: see core/docs/proto-requests-m1.md B3.
|
||||
struct ByteRange {
|
||||
static constexpr std::uint64_t kUnbounded = ~std::uint64_t{0};
|
||||
|
||||
std::uint64_t first = 0;
|
||||
std::uint64_t last = kUnbounded;
|
||||
|
||||
[[nodiscard]] bool bounded() const noexcept { return last != kUnbounded; }
|
||||
[[nodiscard]] std::optional<std::uint64_t> length() const noexcept {
|
||||
if (!bounded())
|
||||
return std::nullopt;
|
||||
return last - first + 1;
|
||||
}
|
||||
// "bytes=100-199" or "bytes=100-"
|
||||
[[nodiscard]] std::string to_header_value() const {
|
||||
std::string v = "bytes=";
|
||||
v += std::to_string(first);
|
||||
v += '-';
|
||||
if (bounded())
|
||||
v += std::to_string(last);
|
||||
return v;
|
||||
}
|
||||
};
|
||||
|
||||
struct HeaderField {
|
||||
std::string name;
|
||||
std::string value;
|
||||
};
|
||||
|
||||
// Case-insensitive view over response headers. Not a multimap for perf — the header set
|
||||
// on a download response is tiny; linear scan is fine and keeps this allocation-light.
|
||||
class HeaderList {
|
||||
public:
|
||||
void add(std::string name, std::string value) {
|
||||
fields_.push_back({std::move(name), std::move(value)});
|
||||
}
|
||||
void clear() noexcept { fields_.clear(); }
|
||||
|
||||
[[nodiscard]] const std::vector<HeaderField> &fields() const noexcept { return fields_; }
|
||||
[[nodiscard]] bool empty() const noexcept { return fields_.empty(); }
|
||||
|
||||
// First value for `name` (ASCII case-insensitive), or nullopt.
|
||||
[[nodiscard]] std::optional<std::string_view> get(std::string_view name) const {
|
||||
for (const auto &f : fields_)
|
||||
if (iequals(f.name, name))
|
||||
return f.value;
|
||||
return std::nullopt;
|
||||
}
|
||||
[[nodiscard]] bool has(std::string_view name) const { return get(name).has_value(); }
|
||||
|
||||
static bool iequals(std::string_view a, std::string_view b) noexcept {
|
||||
if (a.size() != b.size())
|
||||
return false;
|
||||
for (std::size_t i = 0; i < a.size(); ++i)
|
||||
if (lower(a[i]) != lower(b[i]))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
static constexpr char lower(char c) noexcept {
|
||||
return (c >= 'A' && c <= 'Z') ? char(c - 'A' + 'a') : c;
|
||||
}
|
||||
std::vector<HeaderField> fields_;
|
||||
};
|
||||
|
||||
enum class ProxyKind { none, http, socks5, socks5_hostname };
|
||||
|
||||
struct ProxyConfig {
|
||||
ProxyKind kind = ProxyKind::none;
|
||||
std::string host; // host[:port]
|
||||
std::uint16_t port = 0;
|
||||
std::string username; // empty = no proxy auth
|
||||
std::string password;
|
||||
};
|
||||
|
||||
enum class AuthScheme { none, basic, digest, any };
|
||||
|
||||
struct AuthConfig {
|
||||
AuthScheme scheme = AuthScheme::none;
|
||||
std::string username;
|
||||
std::string password;
|
||||
};
|
||||
|
||||
struct Cookie {
|
||||
std::string name;
|
||||
std::string value;
|
||||
};
|
||||
|
||||
// One HTTP request the engine wants performed. Defaults are the well-behaved case.
|
||||
struct Request {
|
||||
std::string url;
|
||||
Method method = Method::get;
|
||||
std::vector<HeaderField> headers; // verbatim; browser UA/Referer/cookies live here
|
||||
std::optional<ByteRange> range;
|
||||
std::vector<Cookie> cookies; // merged into a Cookie: header + curl's jar
|
||||
std::string user_agent; // convenience; also settable via headers
|
||||
std::string referrer;
|
||||
|
||||
ProxyConfig proxy;
|
||||
AuthConfig auth;
|
||||
|
||||
bool follow_redirects = true;
|
||||
long max_redirects = 20;
|
||||
bool accept_encoding = false; // OFF for downloads: a gzip'd body breaks Range math
|
||||
|
||||
// Stall detection: abort if throughput stays under `low_speed_bytes_per_sec` for
|
||||
// `low_speed_secs`. 0 disables. Distinct from an overall deadline (probe sets one).
|
||||
long connect_timeout_ms = 15000;
|
||||
long low_speed_bytes_per_sec = 1024;
|
||||
long low_speed_secs = 30;
|
||||
long overall_timeout_ms = 0; // 0 = none; probe uses a few seconds
|
||||
|
||||
// Coarse download-rate ceiling handed to curl (CURLOPT_MAX_RECV_SPEED_LARGE). The
|
||||
// precise limiter (stage 7) pauses/resumes on top of this. 0 = unlimited.
|
||||
std::uint64_t max_recv_bytes_per_sec = 0;
|
||||
};
|
||||
|
||||
// Delivered once, when response headers are in.
|
||||
struct ResponseHead {
|
||||
long status = 0;
|
||||
std::string effective_url; // after redirects
|
||||
HeaderList headers;
|
||||
std::optional<std::uint64_t> content_length; // from Content-Length, if present & sane
|
||||
};
|
||||
|
||||
} // namespace vdm::net
|
||||
|
||||
#endif // VDM_NET_HTTP_TYPES_HPP
|
||||
@@ -0,0 +1,89 @@
|
||||
// vdm/net/curl_error.cpp
|
||||
|
||||
#include "net/curl_error.hpp"
|
||||
|
||||
namespace vdm::net::detail {
|
||||
|
||||
Error error_from_curl(CURLcode code, long http_status) noexcept {
|
||||
// Transport-level failures first — these override any status.
|
||||
switch (code) {
|
||||
case CURLE_OK:
|
||||
break;
|
||||
case CURLE_COULDNT_RESOLVE_PROXY:
|
||||
case CURLE_COULDNT_RESOLVE_HOST:
|
||||
return Error::resolve_failed;
|
||||
case CURLE_COULDNT_CONNECT:
|
||||
case CURLE_INTERFACE_FAILED:
|
||||
return Error::connect_failed;
|
||||
case CURLE_OPERATION_TIMEDOUT:
|
||||
return Error::timeout;
|
||||
case CURLE_TOO_MANY_REDIRECTS:
|
||||
return Error::too_many_redirects;
|
||||
case CURLE_PEER_FAILED_VERIFICATION:
|
||||
case CURLE_SSL_CONNECT_ERROR:
|
||||
case CURLE_SSL_CERTPROBLEM:
|
||||
case CURLE_SSL_CIPHER:
|
||||
case CURLE_SSL_CACERT_BADFILE:
|
||||
case CURLE_SSL_ISSUER_ERROR:
|
||||
case CURLE_SSL_PINNEDPUBKEYNOTMATCH:
|
||||
case CURLE_SSL_INVALIDCERTSTATUS:
|
||||
return Error::tls_failed;
|
||||
case CURLE_GOT_NOTHING:
|
||||
case CURLE_RECV_ERROR:
|
||||
case CURLE_SEND_ERROR:
|
||||
case CURLE_PARTIAL_FILE:
|
||||
case CURLE_HTTP2:
|
||||
case CURLE_HTTP2_STREAM:
|
||||
return Error::connection_reset;
|
||||
case CURLE_WRITE_ERROR:
|
||||
// Our write callback returned short — the sink (disk) failed or we're
|
||||
// cancelling. The caller distinguishes; default to io_error.
|
||||
return Error::io_error;
|
||||
case CURLE_LOGIN_DENIED:
|
||||
return Error::auth_required;
|
||||
case CURLE_UNSUPPORTED_PROTOCOL:
|
||||
case CURLE_URL_MALFORMAT:
|
||||
return Error::unsupported_url_scheme;
|
||||
case CURLE_ABORTED_BY_CALLBACK:
|
||||
return Error::canceled;
|
||||
default:
|
||||
// Fall through to status-based classification, else generic.
|
||||
break;
|
||||
}
|
||||
|
||||
// HTTP status classification (also reached on CURLE_OK).
|
||||
if (http_status >= 400) {
|
||||
switch (http_status) {
|
||||
case 401:
|
||||
case 407:
|
||||
return Error::auth_required;
|
||||
case 403:
|
||||
return Error::forbidden;
|
||||
case 404:
|
||||
return Error::not_found;
|
||||
case 410:
|
||||
return Error::gone;
|
||||
case 416:
|
||||
return Error::range_not_satisfiable;
|
||||
default:
|
||||
return http_status >= 500 ? Error::http_server_error : Error::http_client_error;
|
||||
}
|
||||
}
|
||||
|
||||
if (code != CURLE_OK)
|
||||
return Error::internal;
|
||||
return Error::ok;
|
||||
}
|
||||
|
||||
ErrorInfo make_error(CURLcode code, long http_status, const char *curl_msg) {
|
||||
Error e = error_from_curl(code, http_status);
|
||||
std::string ctx;
|
||||
if (curl_msg && *curl_msg)
|
||||
ctx = curl_msg;
|
||||
else if (code != CURLE_OK)
|
||||
ctx = curl_easy_strerror(code);
|
||||
ErrorInfo info(e, std::move(ctx), static_cast<int>(http_status));
|
||||
return info;
|
||||
}
|
||||
|
||||
} // namespace vdm::net::detail
|
||||
@@ -0,0 +1,24 @@
|
||||
// vdm/net/curl_error.hpp — internal: CURLcode -> vdm::Error. Not a public header.
|
||||
|
||||
#ifndef VDM_NET_CURL_ERROR_HPP
|
||||
#define VDM_NET_CURL_ERROR_HPP
|
||||
|
||||
#include <curl/curl.h>
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "vdm/util/error.hpp"
|
||||
|
||||
namespace vdm::net::detail {
|
||||
|
||||
// Map a libcurl transfer result to the engine taxonomy. `http_status` (0 if none) lets
|
||||
// the HTTP-status errors (403/404/416/...) be classified here too; pass it from
|
||||
// CURLINFO_RESPONSE_CODE. `CURLE_OK` with a >= 400 status still yields an error.
|
||||
[[nodiscard]] Error error_from_curl(CURLcode code, long http_status) noexcept;
|
||||
|
||||
// A human-readable ErrorInfo, folding in curl's own message and the status.
|
||||
[[nodiscard]] ErrorInfo make_error(CURLcode code, long http_status, const char *curl_msg = nullptr);
|
||||
|
||||
} // namespace vdm::net::detail
|
||||
|
||||
#endif // VDM_NET_CURL_ERROR_HPP
|
||||
@@ -0,0 +1,570 @@
|
||||
// vdm/net/http_client.cpp — libcurl multi implementation.
|
||||
//
|
||||
// Threading model: one Worker == one std::jthread + one CURLM. An easy handle is created,
|
||||
// used, paused, and destroyed only on its Worker's thread. Public calls (start / pause /
|
||||
// resume / cancel) just enqueue a Command and curl_multi_wakeup() the worker.
|
||||
|
||||
#include "vdm/net/http_client.hpp"
|
||||
|
||||
#include <curl/curl.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <charconv>
|
||||
#include <deque>
|
||||
#include <mutex>
|
||||
#include <string_view>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include "net/curl_error.hpp"
|
||||
#include "vdm/util/log.hpp"
|
||||
|
||||
namespace vdm::net {
|
||||
namespace {
|
||||
|
||||
struct CurlGlobal {
|
||||
CurlGlobal() {
|
||||
if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK)
|
||||
VDM_LOG_ERROR("net", "curl_global_init failed");
|
||||
}
|
||||
~CurlGlobal() { curl_global_cleanup(); }
|
||||
};
|
||||
void ensure_curl_global() {
|
||||
static CurlGlobal g;
|
||||
(void)g;
|
||||
}
|
||||
|
||||
std::uint64_t next_id() {
|
||||
static std::atomic<std::uint64_t> counter{0};
|
||||
return ++counter;
|
||||
}
|
||||
|
||||
bool parse_header_line(std::string_view line, std::string &name, std::string &value) {
|
||||
while (!line.empty() && (line.back() == '\r' || line.back() == '\n'))
|
||||
line.remove_suffix(1);
|
||||
if (line.empty())
|
||||
return false;
|
||||
auto colon = line.find(':');
|
||||
if (colon == std::string_view::npos)
|
||||
return false;
|
||||
name.assign(line.substr(0, colon));
|
||||
auto v = line.substr(colon + 1);
|
||||
while (!v.empty() && (v.front() == ' ' || v.front() == '\t'))
|
||||
v.remove_prefix(1);
|
||||
value.assign(v);
|
||||
return true;
|
||||
}
|
||||
|
||||
// "HTTP/1.1 206 Partial Content" -> 206; 0 on parse failure.
|
||||
long status_from_line(std::string_view line) {
|
||||
auto sp = line.find(' ');
|
||||
if (sp == std::string_view::npos)
|
||||
return 0;
|
||||
auto rest = line.substr(sp + 1);
|
||||
long code = 0;
|
||||
auto [p, ec] = std::from_chars(rest.data(), rest.data() + rest.size(), code);
|
||||
(void)p;
|
||||
return ec == std::errc{} ? code : 0;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// --- Transfer::State -----------------------------------------------------------------
|
||||
struct Transfer::State {
|
||||
enum class Stop { none, head_complete, aborted };
|
||||
|
||||
std::uint64_t id = 0;
|
||||
struct HttpClient::Impl *client = nullptr;
|
||||
unsigned worker_index = 0;
|
||||
|
||||
Request req;
|
||||
TransferCallbacks cbs;
|
||||
|
||||
// Worker-thread-owned.
|
||||
CURL *easy = nullptr;
|
||||
curl_slist *header_slist = nullptr;
|
||||
std::string range_value;
|
||||
std::string cookie_value;
|
||||
|
||||
ResponseHead head;
|
||||
long line_status = 0; // status from the most recent HTTP/ line
|
||||
bool head_delivered = false;
|
||||
|
||||
std::atomic<bool> pause_requested{false};
|
||||
std::atomic<Stop> stop{Stop::none};
|
||||
bool curl_paused = false;
|
||||
|
||||
std::uint64_t bytes_received = 0;
|
||||
bool finished = false;
|
||||
};
|
||||
|
||||
// --- Impl -------------------------------------------------------------------------
|
||||
struct HttpClient::Impl {
|
||||
enum class CmdKind { add, pause, resume, cancel };
|
||||
struct Command {
|
||||
CmdKind kind;
|
||||
std::shared_ptr<Transfer::State> state;
|
||||
};
|
||||
|
||||
struct Worker {
|
||||
CURLM *multi = nullptr;
|
||||
std::mutex mu;
|
||||
std::deque<Command> queue;
|
||||
std::vector<std::shared_ptr<Transfer::State>> live;
|
||||
std::jthread thread;
|
||||
};
|
||||
|
||||
explicit Impl(Options o) : opts(o) {
|
||||
ensure_curl_global();
|
||||
|
||||
unsigned n = opts.workers;
|
||||
if (n == 0) {
|
||||
unsigned hw = std::thread::hardware_concurrency();
|
||||
n = std::clamp<unsigned>(hw ? hw : 1, 1, 4);
|
||||
}
|
||||
|
||||
if (opts.share_dns_and_tls) {
|
||||
share = curl_share_init();
|
||||
if (share) {
|
||||
curl_share_setopt(share, CURLSHOPT_LOCKFUNC, &Impl::share_lock);
|
||||
curl_share_setopt(share, CURLSHOPT_UNLOCKFUNC, &Impl::share_unlock);
|
||||
curl_share_setopt(share, CURLSHOPT_USERDATA, this);
|
||||
curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
|
||||
curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
|
||||
}
|
||||
}
|
||||
|
||||
workers.reserve(n);
|
||||
for (unsigned i = 0; i < n; ++i) {
|
||||
auto w = std::make_unique<Worker>();
|
||||
w->multi = curl_multi_init();
|
||||
if (opts.max_connections_per_worker > 0)
|
||||
curl_multi_setopt(w->multi, CURLMOPT_MAX_TOTAL_CONNECTIONS,
|
||||
opts.max_connections_per_worker);
|
||||
Worker *raw = w.get();
|
||||
w->thread = std::jthread([this, raw](std::stop_token st) { run(*raw, st); });
|
||||
workers.push_back(std::move(w));
|
||||
}
|
||||
}
|
||||
|
||||
~Impl() {
|
||||
stopping.store(true);
|
||||
for (auto &w : workers) {
|
||||
w->thread.request_stop();
|
||||
if (w->multi)
|
||||
curl_multi_wakeup(w->multi);
|
||||
}
|
||||
for (auto &w : workers)
|
||||
if (w->thread.joinable())
|
||||
w->thread.join();
|
||||
for (auto &w : workers)
|
||||
if (w->multi)
|
||||
curl_multi_cleanup(w->multi);
|
||||
if (share)
|
||||
curl_share_cleanup(share);
|
||||
}
|
||||
|
||||
Options opts;
|
||||
std::vector<std::unique_ptr<Worker>> workers;
|
||||
CURLSH *share = nullptr;
|
||||
std::mutex share_mu[CURL_LOCK_DATA_LAST];
|
||||
std::atomic<unsigned> rr{0};
|
||||
std::atomic<bool> stopping{false};
|
||||
|
||||
static void share_lock(CURL *, curl_lock_data data, curl_lock_access, void *userp) {
|
||||
static_cast<Impl *>(userp)->share_mu[data].lock();
|
||||
}
|
||||
static void share_unlock(CURL *, curl_lock_data data, void *userp) {
|
||||
static_cast<Impl *>(userp)->share_mu[data].unlock();
|
||||
}
|
||||
|
||||
void enqueue(unsigned wi, Command cmd) {
|
||||
Worker &w = *workers[wi];
|
||||
{
|
||||
std::lock_guard lk(w.mu);
|
||||
w.queue.push_back(std::move(cmd));
|
||||
}
|
||||
curl_multi_wakeup(w.multi);
|
||||
}
|
||||
|
||||
// ---- curl C callbacks ----
|
||||
static std::size_t header_cb(char *buf, std::size_t size, std::size_t n, void *userp) {
|
||||
auto *st = static_cast<Transfer::State *>(userp);
|
||||
const std::size_t total = size * n;
|
||||
std::string_view line(buf, total);
|
||||
|
||||
if (line.starts_with("HTTP/")) {
|
||||
st->line_status = status_from_line(line);
|
||||
st->head.headers.clear(); // keep only the final response's headers
|
||||
return total;
|
||||
}
|
||||
if (line == "\r\n" || line == "\n") {
|
||||
const bool redirect =
|
||||
st->req.follow_redirects && st->line_status >= 300 && st->line_status < 400;
|
||||
if (!redirect)
|
||||
deliver_head(st);
|
||||
return total;
|
||||
}
|
||||
std::string name, value;
|
||||
if (parse_header_line(line, name, value))
|
||||
st->head.headers.add(std::move(name), std::move(value));
|
||||
return total;
|
||||
}
|
||||
|
||||
static void deliver_head(Transfer::State *st) {
|
||||
if (st->head_delivered)
|
||||
return;
|
||||
st->head_delivered = true;
|
||||
|
||||
long code = 0;
|
||||
curl_easy_getinfo(st->easy, CURLINFO_RESPONSE_CODE, &code);
|
||||
st->head.status = code ? code : st->line_status;
|
||||
char *eff = nullptr;
|
||||
if (curl_easy_getinfo(st->easy, CURLINFO_EFFECTIVE_URL, &eff) == CURLE_OK && eff)
|
||||
st->head.effective_url = eff;
|
||||
curl_off_t clen = -1;
|
||||
if (curl_easy_getinfo(st->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &clen) == CURLE_OK &&
|
||||
clen >= 0)
|
||||
st->head.content_length = static_cast<std::uint64_t>(clen);
|
||||
|
||||
if (st->cbs.on_head) {
|
||||
DataAction a = st->cbs.on_head(st->head);
|
||||
if (a == DataAction::abort)
|
||||
st->stop.store(Transfer::State::Stop::head_complete);
|
||||
else if (a == DataAction::pause)
|
||||
st->pause_requested.store(true);
|
||||
}
|
||||
}
|
||||
|
||||
static std::size_t write_cb(char *ptr, std::size_t size, std::size_t n, void *userp) {
|
||||
auto *st = static_cast<Transfer::State *>(userp);
|
||||
const std::size_t total = size * n;
|
||||
|
||||
if (!st->head_delivered)
|
||||
deliver_head(st);
|
||||
if (st->stop.load() != Transfer::State::Stop::none)
|
||||
return 0; // -> CURLE_WRITE_ERROR
|
||||
if (st->pause_requested.load()) {
|
||||
st->curl_paused = true;
|
||||
return CURL_WRITEFUNC_PAUSE;
|
||||
}
|
||||
|
||||
if (total && st->cbs.on_data) {
|
||||
ConstByteSpan span(reinterpret_cast<const std::byte *>(ptr), total);
|
||||
DataAction a = st->cbs.on_data(span);
|
||||
if (a == DataAction::abort) {
|
||||
st->stop.store(Transfer::State::Stop::aborted);
|
||||
return 0;
|
||||
}
|
||||
if (a == DataAction::pause) {
|
||||
st->pause_requested.store(true);
|
||||
st->curl_paused = true;
|
||||
return CURL_WRITEFUNC_PAUSE;
|
||||
}
|
||||
}
|
||||
st->bytes_received += total;
|
||||
return total;
|
||||
}
|
||||
|
||||
// ---- worker thread ----
|
||||
void run(Worker &w, std::stop_token stok) {
|
||||
while (!stok.stop_requested()) {
|
||||
drain_commands(w);
|
||||
int running = 0;
|
||||
curl_multi_perform(w.multi, &running);
|
||||
reap(w);
|
||||
if (stok.stop_requested())
|
||||
break;
|
||||
int numfds = 0;
|
||||
curl_multi_poll(w.multi, nullptr, 0, 1000, &numfds);
|
||||
}
|
||||
shutdown_worker(w);
|
||||
}
|
||||
|
||||
void drain_commands(Worker &w) {
|
||||
std::deque<Command> local;
|
||||
{
|
||||
std::lock_guard lk(w.mu);
|
||||
local.swap(w.queue);
|
||||
}
|
||||
for (auto &cmd : local) {
|
||||
auto &st = cmd.state;
|
||||
switch (cmd.kind) {
|
||||
case CmdKind::add:
|
||||
attach(w, st);
|
||||
break;
|
||||
case CmdKind::pause:
|
||||
st->pause_requested.store(true);
|
||||
if (st->easy && !st->curl_paused) {
|
||||
curl_easy_pause(st->easy, CURLPAUSE_RECV);
|
||||
st->curl_paused = true;
|
||||
}
|
||||
break;
|
||||
case CmdKind::resume:
|
||||
st->pause_requested.store(false);
|
||||
if (st->easy && st->curl_paused) {
|
||||
st->curl_paused = false;
|
||||
curl_easy_pause(st->easy, CURLPAUSE_CONT);
|
||||
}
|
||||
break;
|
||||
case CmdKind::cancel:
|
||||
st->stop.store(Transfer::State::Stop::aborted);
|
||||
if (st->easy && st->curl_paused) {
|
||||
st->curl_paused = false;
|
||||
curl_easy_pause(st->easy, CURLPAUSE_CONT); // let write_cb return 0
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void attach(Worker &w, std::shared_ptr<Transfer::State> st) {
|
||||
if (stopping.load()) {
|
||||
complete(st, ErrorInfo(Error::canceled, "client shutting down"));
|
||||
return;
|
||||
}
|
||||
CURL *e = curl_easy_init();
|
||||
if (!e) {
|
||||
complete(st, ErrorInfo(Error::internal, "curl_easy_init"));
|
||||
return;
|
||||
}
|
||||
st->easy = e;
|
||||
const Request &r = st->req;
|
||||
|
||||
curl_easy_setopt(e, CURLOPT_URL, r.url.c_str());
|
||||
curl_easy_setopt(e, CURLOPT_PRIVATE, st.get());
|
||||
curl_easy_setopt(e, CURLOPT_NOSIGNAL, 1L);
|
||||
curl_easy_setopt(e, CURLOPT_NOPROGRESS, 1L);
|
||||
curl_easy_setopt(e, CURLOPT_HEADERFUNCTION, &Impl::header_cb);
|
||||
curl_easy_setopt(e, CURLOPT_HEADERDATA, st.get());
|
||||
curl_easy_setopt(e, CURLOPT_WRITEFUNCTION, &Impl::write_cb);
|
||||
curl_easy_setopt(e, CURLOPT_WRITEDATA, st.get());
|
||||
curl_easy_setopt(e, CURLOPT_TCP_KEEPALIVE, 1L);
|
||||
if (share)
|
||||
curl_easy_setopt(e, CURLOPT_SHARE, share);
|
||||
|
||||
if (r.method == Method::head)
|
||||
curl_easy_setopt(e, CURLOPT_NOBODY, 1L);
|
||||
|
||||
if (r.follow_redirects) {
|
||||
curl_easy_setopt(e, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(e, CURLOPT_MAXREDIRS, r.max_redirects);
|
||||
}
|
||||
curl_easy_setopt(e, CURLOPT_ACCEPT_ENCODING, r.accept_encoding ? "" : nullptr);
|
||||
|
||||
if (r.range) {
|
||||
std::string v = r.range->to_header_value(); // "bytes=first-last"
|
||||
std::string_view sv = v;
|
||||
if (sv.starts_with("bytes="))
|
||||
sv.remove_prefix(6);
|
||||
st->range_value.assign(sv);
|
||||
curl_easy_setopt(e, CURLOPT_RANGE, st->range_value.c_str());
|
||||
}
|
||||
|
||||
curl_easy_setopt(e, CURLOPT_CONNECTTIMEOUT_MS, static_cast<long>(r.connect_timeout_ms));
|
||||
if (r.overall_timeout_ms > 0)
|
||||
curl_easy_setopt(e, CURLOPT_TIMEOUT_MS, static_cast<long>(r.overall_timeout_ms));
|
||||
if (r.low_speed_bytes_per_sec > 0 && r.low_speed_secs > 0) {
|
||||
curl_easy_setopt(e, CURLOPT_LOW_SPEED_LIMIT, r.low_speed_bytes_per_sec);
|
||||
curl_easy_setopt(e, CURLOPT_LOW_SPEED_TIME, r.low_speed_secs);
|
||||
}
|
||||
if (r.max_recv_bytes_per_sec > 0)
|
||||
curl_easy_setopt(e, CURLOPT_MAX_RECV_SPEED_LARGE,
|
||||
static_cast<curl_off_t>(r.max_recv_bytes_per_sec));
|
||||
|
||||
if (!r.user_agent.empty())
|
||||
curl_easy_setopt(e, CURLOPT_USERAGENT, r.user_agent.c_str());
|
||||
if (!r.referrer.empty())
|
||||
curl_easy_setopt(e, CURLOPT_REFERER, r.referrer.c_str());
|
||||
|
||||
if (r.proxy.kind != ProxyKind::none) {
|
||||
curl_easy_setopt(e, CURLOPT_PROXY, r.proxy.host.c_str());
|
||||
if (r.proxy.port)
|
||||
curl_easy_setopt(e, CURLOPT_PROXYPORT, long(r.proxy.port));
|
||||
long pt = CURLPROXY_HTTP;
|
||||
if (r.proxy.kind == ProxyKind::socks5)
|
||||
pt = CURLPROXY_SOCKS5;
|
||||
else if (r.proxy.kind == ProxyKind::socks5_hostname)
|
||||
pt = CURLPROXY_SOCKS5_HOSTNAME;
|
||||
curl_easy_setopt(e, CURLOPT_PROXYTYPE, pt);
|
||||
if (!r.proxy.username.empty()) {
|
||||
std::string up = r.proxy.username + ":" + r.proxy.password;
|
||||
curl_easy_setopt(e, CURLOPT_PROXYUSERPWD, up.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
if (r.auth.scheme != AuthScheme::none) {
|
||||
long m = CURLAUTH_ANY;
|
||||
if (r.auth.scheme == AuthScheme::basic)
|
||||
m = CURLAUTH_BASIC;
|
||||
else if (r.auth.scheme == AuthScheme::digest)
|
||||
m = CURLAUTH_DIGEST;
|
||||
curl_easy_setopt(e, CURLOPT_HTTPAUTH, m);
|
||||
std::string up = r.auth.username + ":" + r.auth.password;
|
||||
curl_easy_setopt(e, CURLOPT_USERPWD, up.c_str());
|
||||
}
|
||||
|
||||
if (!r.cookies.empty()) {
|
||||
for (const auto &c : r.cookies) {
|
||||
if (!st->cookie_value.empty())
|
||||
st->cookie_value += "; ";
|
||||
st->cookie_value += c.name + "=" + c.value;
|
||||
}
|
||||
curl_easy_setopt(e, CURLOPT_COOKIE, st->cookie_value.c_str());
|
||||
}
|
||||
|
||||
for (const auto &h : r.headers) {
|
||||
std::string joined = h.name + ": " + h.value;
|
||||
st->header_slist = curl_slist_append(st->header_slist, joined.c_str());
|
||||
}
|
||||
if (st->header_slist)
|
||||
curl_easy_setopt(e, CURLOPT_HTTPHEADER, st->header_slist);
|
||||
|
||||
CURLMcode mc = curl_multi_add_handle(w.multi, e);
|
||||
if (mc != CURLM_OK) {
|
||||
curl_easy_cleanup(e);
|
||||
st->easy = nullptr;
|
||||
complete(st, ErrorInfo(Error::internal, curl_multi_strerror(mc)));
|
||||
return;
|
||||
}
|
||||
w.live.push_back(std::move(st));
|
||||
}
|
||||
|
||||
void reap(Worker &w) {
|
||||
CURLMsg *msg = nullptr;
|
||||
int inq = 0;
|
||||
while ((msg = curl_multi_info_read(w.multi, &inq)) != nullptr) {
|
||||
if (msg->msg != CURLMSG_DONE)
|
||||
continue;
|
||||
CURL *e = msg->easy_handle;
|
||||
const CURLcode res = msg->data.result;
|
||||
|
||||
Transfer::State *raw = nullptr;
|
||||
curl_easy_getinfo(e, CURLINFO_PRIVATE, &raw);
|
||||
long code = 0;
|
||||
curl_easy_getinfo(e, CURLINFO_RESPONSE_CODE, &code);
|
||||
|
||||
TransferStats stats;
|
||||
stats.http_status = code;
|
||||
gather_timings(e, stats);
|
||||
|
||||
auto it = std::find_if(w.live.begin(), w.live.end(),
|
||||
[raw](const auto &s) { return s.get() == raw; });
|
||||
std::shared_ptr<Transfer::State> st = (it != w.live.end()) ? *it : nullptr;
|
||||
|
||||
curl_multi_remove_handle(w.multi, e);
|
||||
curl_easy_cleanup(e);
|
||||
if (st) {
|
||||
st->easy = nullptr;
|
||||
if (st->header_slist) {
|
||||
curl_slist_free_all(st->header_slist);
|
||||
st->header_slist = nullptr;
|
||||
}
|
||||
}
|
||||
if (it != w.live.end())
|
||||
w.live.erase(it);
|
||||
if (!st)
|
||||
continue;
|
||||
|
||||
using Stop = Transfer::State::Stop;
|
||||
const Stop stop = st->stop.load();
|
||||
|
||||
if (stop == Stop::aborted) {
|
||||
complete(st, ErrorInfo(Error::canceled));
|
||||
} else if (stop == Stop::head_complete) {
|
||||
// A probe: on_head asked to stop. Headers were the goal -> success, even
|
||||
// though a ranged GET body-stop surfaces as CURLE_WRITE_ERROR.
|
||||
stats.bytes_received = st->bytes_received;
|
||||
stats.effective_url = st->head.effective_url;
|
||||
complete(st, std::move(stats));
|
||||
} else if (res == CURLE_OK && code < 400) {
|
||||
stats.bytes_received = st->bytes_received;
|
||||
stats.effective_url = st->head.effective_url;
|
||||
complete(st, std::move(stats));
|
||||
} else {
|
||||
complete(st, detail::make_error(res, code));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void gather_timings(CURL *e, TransferStats &s) {
|
||||
auto us_to_ms = [](curl_off_t us) { return us > 0 ? static_cast<long>(us / 1000) : 0L; };
|
||||
curl_off_t t = 0;
|
||||
if (curl_easy_getinfo(e, CURLINFO_NAMELOOKUP_TIME_T, &t) == CURLE_OK)
|
||||
s.namelookup_ms = us_to_ms(t);
|
||||
if (curl_easy_getinfo(e, CURLINFO_CONNECT_TIME_T, &t) == CURLE_OK)
|
||||
s.connect_ms = us_to_ms(t);
|
||||
if (curl_easy_getinfo(e, CURLINFO_APPCONNECT_TIME_T, &t) == CURLE_OK)
|
||||
s.appconnect_ms = us_to_ms(t);
|
||||
if (curl_easy_getinfo(e, CURLINFO_STARTTRANSFER_TIME_T, &t) == CURLE_OK)
|
||||
s.starttransfer_ms = us_to_ms(t);
|
||||
if (curl_easy_getinfo(e, CURLINFO_TOTAL_TIME_T, &t) == CURLE_OK)
|
||||
s.total_ms = us_to_ms(t);
|
||||
}
|
||||
|
||||
void complete(const std::shared_ptr<Transfer::State> &st, Result<TransferStats> r) {
|
||||
if (st->finished)
|
||||
return;
|
||||
st->finished = true;
|
||||
if (st->cbs.on_finished)
|
||||
st->cbs.on_finished(std::move(r));
|
||||
}
|
||||
|
||||
void shutdown_worker(Worker &w) {
|
||||
for (auto &st : w.live) {
|
||||
if (st->easy) {
|
||||
curl_multi_remove_handle(w.multi, st->easy);
|
||||
curl_easy_cleanup(st->easy);
|
||||
st->easy = nullptr;
|
||||
}
|
||||
if (st->header_slist) {
|
||||
curl_slist_free_all(st->header_slist);
|
||||
st->header_slist = nullptr;
|
||||
}
|
||||
complete(st, ErrorInfo(Error::canceled, "client shutting down"));
|
||||
}
|
||||
w.live.clear();
|
||||
}
|
||||
};
|
||||
|
||||
// --- Transfer -------------------------------------------------------------------
|
||||
std::uint64_t Transfer::id() const noexcept {
|
||||
return state_ ? state_->id : 0;
|
||||
}
|
||||
|
||||
void Transfer::pause() {
|
||||
if (state_ && state_->client)
|
||||
state_->client->enqueue(state_->worker_index, {HttpClient::Impl::CmdKind::pause, state_});
|
||||
}
|
||||
void Transfer::resume() {
|
||||
if (state_ && state_->client)
|
||||
state_->client->enqueue(state_->worker_index, {HttpClient::Impl::CmdKind::resume, state_});
|
||||
}
|
||||
void Transfer::cancel() {
|
||||
if (state_ && state_->client)
|
||||
state_->client->enqueue(state_->worker_index, {HttpClient::Impl::CmdKind::cancel, state_});
|
||||
}
|
||||
|
||||
// --- HttpClient ---------------------------------------------------------------
|
||||
HttpClient::HttpClient() : HttpClient(Options{}) {}
|
||||
HttpClient::HttpClient(Options opts) : impl_(std::make_unique<Impl>(opts)) {}
|
||||
HttpClient::~HttpClient() = default;
|
||||
|
||||
unsigned HttpClient::worker_count() const noexcept {
|
||||
return impl_ ? static_cast<unsigned>(impl_->workers.size()) : 0;
|
||||
}
|
||||
|
||||
Transfer HttpClient::start(Request req, TransferCallbacks cbs) {
|
||||
auto st = std::make_shared<Transfer::State>();
|
||||
st->id = next_id();
|
||||
st->client = impl_.get();
|
||||
st->req = std::move(req);
|
||||
st->cbs = std::move(cbs);
|
||||
st->worker_index = impl_->workers.empty() ? 0 : impl_->rr.fetch_add(1) % impl_->workers.size();
|
||||
|
||||
impl_->enqueue(st->worker_index, {Impl::CmdKind::add, st});
|
||||
return Transfer(st);
|
||||
}
|
||||
|
||||
} // namespace vdm::net
|
||||
@@ -20,3 +20,17 @@ vdm_add_test(veloxcore_event_bus_test util/event_bus_test.cpp)
|
||||
vdm_add_test(veloxcore_thread_pool_test util/thread_pool_test.cpp)
|
||||
vdm_add_test(veloxcore_bytes_test util/bytes_test.cpp)
|
||||
vdm_add_test(veloxcore_log_test util/log_test.cpp)
|
||||
|
||||
# net/ integration tests drive tools/testserver (lane PKG/QA). Skip cleanly if it isn't
|
||||
# in the tree yet (lanes merge independently).
|
||||
set(_testserver ${CMAKE_SOURCE_DIR}/tools/testserver/testserver.py)
|
||||
vdm_add_test(veloxcore_http_client_test net/http_client_test.cpp)
|
||||
target_include_directories(veloxcore_http_client_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/net)
|
||||
if(EXISTS ${_testserver})
|
||||
target_compile_definitions(veloxcore_http_client_test
|
||||
PRIVATE VDM_TESTSERVER_PY="${_testserver}")
|
||||
set_tests_properties(veloxcore_http_client_test PROPERTIES TIMEOUT 120)
|
||||
else()
|
||||
message(STATUS "veloxcore: tools/testserver not present; http_client_test will skip "
|
||||
"its server-backed cases.")
|
||||
endif()
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
#include "vdm/net/http_client.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <future>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "testserver_fixture.hpp"
|
||||
#include "vtest.hpp"
|
||||
|
||||
using namespace vdm;
|
||||
using namespace vdm::net;
|
||||
using vdm::testing::TestServer;
|
||||
|
||||
namespace {
|
||||
|
||||
// Collects callback output from a transfer and lets the test thread wait for the end.
|
||||
struct Recorder {
|
||||
std::mutex mu;
|
||||
ResponseHead head;
|
||||
bool head_seen = false;
|
||||
std::uint64_t bytes = 0;
|
||||
std::promise<Result<TransferStats>> done;
|
||||
std::future<Result<TransferStats>> done_fut = done.get_future();
|
||||
|
||||
DataAction want_on_head = DataAction::proceed; // set before start()
|
||||
std::atomic<DataAction> want_on_data{DataAction::proceed};
|
||||
std::atomic<int> data_calls{0};
|
||||
|
||||
TransferCallbacks callbacks() {
|
||||
return TransferCallbacks{
|
||||
.on_head =
|
||||
[this](const ResponseHead &h) {
|
||||
std::lock_guard lk(mu);
|
||||
head = h;
|
||||
head_seen = true;
|
||||
return want_on_head;
|
||||
},
|
||||
.on_data =
|
||||
[this](ConstByteSpan s) {
|
||||
data_calls.fetch_add(1);
|
||||
std::lock_guard lk(mu);
|
||||
bytes += s.size();
|
||||
return want_on_data.load();
|
||||
},
|
||||
.on_finished = [this](Result<TransferStats> r) { done.set_value(std::move(r)); },
|
||||
};
|
||||
}
|
||||
|
||||
Result<TransferStats> wait(std::chrono::seconds to = std::chrono::seconds(20)) {
|
||||
if (done_fut.wait_for(to) != std::future_status::ready)
|
||||
return Err{Error::timeout, "test wait timed out"};
|
||||
return done_fut.get();
|
||||
}
|
||||
};
|
||||
|
||||
// on_data needs an atomic for the cancel/pause tests to flip it from the test thread.
|
||||
struct AtomicAction {
|
||||
std::atomic<DataAction> a{DataAction::proceed};
|
||||
void store(DataAction v) { a.store(v); }
|
||||
operator DataAction() const { return a.load(); }
|
||||
DataAction load() const { return a.load(); }
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
VT_TEST(http_plain_full_get) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
HttpClient client({.workers = 1});
|
||||
|
||||
Recorder rec;
|
||||
Request req;
|
||||
req.url = srv.url("/plain/file/64K");
|
||||
auto t = client.start(std::move(req), rec.callbacks());
|
||||
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(r.value().http_status, 200);
|
||||
VT_CHECK_EQ(r.value().bytes_received, 65536u);
|
||||
VT_CHECK(rec.head_seen);
|
||||
VT_CHECK_EQ(rec.head.status, 200);
|
||||
VT_CHECK(rec.head.headers.has("Accept-Ranges"));
|
||||
VT_CHECK_EQ(rec.bytes, 65536u);
|
||||
VT_CHECK(t.id() != 0);
|
||||
}
|
||||
|
||||
VT_TEST(http_ranged_get_is_206) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
HttpClient client({.workers = 1});
|
||||
|
||||
Recorder rec;
|
||||
Request req;
|
||||
req.url = srv.url("/plain/file/64K");
|
||||
req.range = ByteRange{1000, 1999};
|
||||
auto t = client.start(std::move(req), rec.callbacks());
|
||||
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(rec.head.status, 206);
|
||||
auto cr = rec.head.headers.get("Content-Range");
|
||||
VT_REQUIRE(cr.has_value());
|
||||
VT_CHECK_EQ(std::string(*cr), std::string("bytes 1000-1999/65536"));
|
||||
VT_CHECK_EQ(rec.bytes, 1000u);
|
||||
VT_CHECK_EQ(r.value().bytes_received, 1000u);
|
||||
}
|
||||
|
||||
VT_TEST(http_follows_redirect_chain) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
HttpClient client({.workers = 1});
|
||||
|
||||
Recorder rec;
|
||||
Request req;
|
||||
req.url = srv.url("/redirect-chain/file/16K");
|
||||
auto t = client.start(std::move(req), rec.callbacks());
|
||||
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(rec.head.status, 200); // on_head fires once, for the final response
|
||||
VT_CHECK_EQ(rec.bytes, 16u * 1024u);
|
||||
VT_CHECK(r.value().effective_url != srv.url("/redirect-chain/file/16K"));
|
||||
VT_CHECK(r.value().effective_url.find("_r=done") != std::string::npos);
|
||||
}
|
||||
|
||||
VT_TEST(http_404_is_not_found_error) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
HttpClient client({.workers = 1});
|
||||
|
||||
Recorder rec;
|
||||
Request req;
|
||||
req.url = srv.url("/plain/nope");
|
||||
auto t = client.start(std::move(req), rec.callbacks());
|
||||
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::not_found);
|
||||
VT_CHECK_EQ(r.error().http_status, 404);
|
||||
}
|
||||
|
||||
VT_TEST(http_connection_refused_is_connect_failed) {
|
||||
HttpClient client({.workers = 1});
|
||||
Recorder rec;
|
||||
Request req;
|
||||
req.url = "http://127.0.0.1:1/nothing"; // nothing listens on :1
|
||||
req.connect_timeout_ms = 2000;
|
||||
auto t = client.start(std::move(req), rec.callbacks());
|
||||
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::connect_failed);
|
||||
}
|
||||
|
||||
VT_TEST(http_head_probe_stops_after_headers) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
HttpClient client({.workers = 1});
|
||||
|
||||
Recorder rec;
|
||||
rec.want_on_head = DataAction::abort; // probe: headers only
|
||||
Request req;
|
||||
req.url = srv.url("/plain/file/1M");
|
||||
req.method = Method::head;
|
||||
auto t = client.start(std::move(req), rec.callbacks());
|
||||
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(r.has_value()); // head_complete, NOT canceled
|
||||
VT_CHECK(rec.head_seen);
|
||||
VT_REQUIRE(rec.head.content_length.has_value());
|
||||
VT_CHECK_EQ(*rec.head.content_length, 1024u * 1024u);
|
||||
VT_CHECK_EQ(rec.data_calls.load(), 0);
|
||||
}
|
||||
|
||||
VT_TEST(http_ranged_get_probe_stops_without_downloading_file) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
HttpClient client({.workers = 1});
|
||||
|
||||
Recorder rec;
|
||||
rec.want_on_head = DataAction::abort;
|
||||
Request req;
|
||||
req.url = srv.url("/plain/file/8M");
|
||||
req.range = ByteRange{0, 0}; // classic HEAD-refused fallback
|
||||
auto t = client.start(std::move(req), rec.callbacks());
|
||||
|
||||
auto r = rec.wait();
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK(rec.bytes <= 1u); // at most the one probe byte, usually 0
|
||||
}
|
||||
|
||||
VT_TEST(http_cancel_mid_transfer) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
HttpClient client({.workers = 1});
|
||||
|
||||
AtomicAction data_action;
|
||||
std::promise<Result<TransferStats>> done;
|
||||
auto fut = done.get_future();
|
||||
std::atomic<int> calls{0};
|
||||
|
||||
TransferCallbacks cbs{
|
||||
.on_head = [](const ResponseHead &) { return DataAction::proceed; },
|
||||
.on_data =
|
||||
[&](ConstByteSpan) {
|
||||
calls.fetch_add(1);
|
||||
return data_action.load();
|
||||
},
|
||||
.on_finished = [&](Result<TransferStats> r) { done.set_value(std::move(r)); },
|
||||
};
|
||||
|
||||
Request req;
|
||||
req.url = srv.url("/throttled/file/4M"); // ~128 KiB/s => lots of chunks
|
||||
auto t = client.start(std::move(req), std::move(cbs));
|
||||
|
||||
// wait for the transfer to actually start, then cancel
|
||||
for (int i = 0; i < 200 && calls.load() == 0; ++i)
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(10));
|
||||
VT_REQUIRE(calls.load() > 0);
|
||||
t.cancel();
|
||||
|
||||
VT_REQUIRE(fut.wait_for(std::chrono::seconds(10)) == std::future_status::ready);
|
||||
auto r = fut.get();
|
||||
VT_REQUIRE(!r.has_value());
|
||||
VT_CHECK_EQ(r.error().code, Error::canceled);
|
||||
}
|
||||
|
||||
VT_TEST(http_pause_then_resume_completes) {
|
||||
TestServer srv;
|
||||
VT_REQUIRE(srv.available());
|
||||
HttpClient client({.workers = 1});
|
||||
|
||||
std::atomic<int> calls{0};
|
||||
std::atomic<std::uint64_t> total{0};
|
||||
std::promise<Result<TransferStats>> done;
|
||||
auto fut = done.get_future();
|
||||
|
||||
TransferCallbacks cbs{
|
||||
.on_head = [](const ResponseHead &) { return DataAction::proceed; },
|
||||
.on_data =
|
||||
[&](ConstByteSpan s) {
|
||||
calls.fetch_add(1);
|
||||
total.fetch_add(s.size());
|
||||
return DataAction::proceed;
|
||||
},
|
||||
.on_finished = [&](Result<TransferStats> r) { done.set_value(std::move(r)); },
|
||||
};
|
||||
|
||||
Request req;
|
||||
req.url = srv.url("/throttled/file/1M");
|
||||
auto t = client.start(std::move(req), std::move(cbs));
|
||||
|
||||
for (int i = 0; i < 200 && calls.load() == 0; ++i)
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(10));
|
||||
VT_REQUIRE(calls.load() > 0);
|
||||
|
||||
t.pause();
|
||||
int calls_at_pause = calls.load();
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(400));
|
||||
VT_CHECK(calls.load() - calls_at_pause <= 1); // at most one in-flight chunk slips through
|
||||
t.resume();
|
||||
|
||||
VT_REQUIRE(fut.wait_for(std::chrono::seconds(20)) == std::future_status::ready);
|
||||
auto r = fut.get();
|
||||
VT_REQUIRE(r.has_value());
|
||||
VT_CHECK_EQ(total.load(), 1024u * 1024u);
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
// testserver_fixture.hpp — spawn tools/testserver for a test, tear it down after.
|
||||
//
|
||||
// Linux-only (fork/exec/pipe/kill). The path to testserver.py is injected by CMake as
|
||||
// VDM_TESTSERVER_PY; if it's empty or missing the fixture reports unavailable() and the
|
||||
// test should skip.
|
||||
|
||||
#ifndef VDM_TESTS_NET_TESTSERVER_FIXTURE_HPP
|
||||
#define VDM_TESTS_NET_TESTSERVER_FIXTURE_HPP
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <signal.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cerrno>
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
|
||||
#ifndef VDM_TESTSERVER_PY
|
||||
#define VDM_TESTSERVER_PY ""
|
||||
#endif
|
||||
|
||||
namespace vdm::testing {
|
||||
|
||||
class TestServer {
|
||||
public:
|
||||
TestServer() {
|
||||
const char *script = VDM_TESTSERVER_PY;
|
||||
if (!script || !*script || ::access(script, R_OK) != 0)
|
||||
return;
|
||||
|
||||
int pipefd[2];
|
||||
if (::pipe(pipefd) != 0)
|
||||
return;
|
||||
|
||||
pid_ = ::fork();
|
||||
if (pid_ < 0) {
|
||||
::close(pipefd[0]);
|
||||
::close(pipefd[1]);
|
||||
return;
|
||||
}
|
||||
if (pid_ == 0) {
|
||||
::dup2(pipefd[1], STDOUT_FILENO);
|
||||
::close(pipefd[0]);
|
||||
::close(pipefd[1]);
|
||||
int devnull = ::open("/dev/null", O_WRONLY);
|
||||
if (devnull >= 0)
|
||||
::dup2(devnull, STDERR_FILENO);
|
||||
::execlp("python3", "python3", script, "--port", "0", "--seed", "9", "--loris-seconds",
|
||||
"1", "--throttle-bps", "131072", static_cast<char *>(nullptr));
|
||||
::_exit(127);
|
||||
}
|
||||
::close(pipefd[1]);
|
||||
|
||||
// Read the port line the server prints to stdout.
|
||||
std::string line;
|
||||
char c = 0;
|
||||
auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10);
|
||||
while (std::chrono::steady_clock::now() < deadline) {
|
||||
ssize_t r = ::read(pipefd[0], &c, 1);
|
||||
if (r == 1) {
|
||||
if (c == '\n')
|
||||
break;
|
||||
line += c;
|
||||
} else if (r == 0) {
|
||||
break;
|
||||
} else if (errno != EINTR) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
::close(pipefd[0]);
|
||||
if (!line.empty())
|
||||
port_ = std::atoi(line.c_str());
|
||||
|
||||
// Give the listener a moment to accept.
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(150));
|
||||
}
|
||||
|
||||
~TestServer() {
|
||||
if (pid_ > 0) {
|
||||
::kill(pid_, SIGTERM);
|
||||
int status = 0;
|
||||
for (int i = 0; i < 50; ++i) {
|
||||
if (::waitpid(pid_, &status, WNOHANG) == pid_)
|
||||
return;
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(20));
|
||||
}
|
||||
::kill(pid_, SIGKILL);
|
||||
::waitpid(pid_, &status, 0);
|
||||
}
|
||||
}
|
||||
|
||||
TestServer(const TestServer &) = delete;
|
||||
TestServer &operator=(const TestServer &) = delete;
|
||||
|
||||
[[nodiscard]] bool available() const { return port_ > 0; }
|
||||
[[nodiscard]] int port() const { return port_; }
|
||||
[[nodiscard]] std::string url(const std::string &path) const {
|
||||
return "http://127.0.0.1:" + std::to_string(port_) + path;
|
||||
}
|
||||
|
||||
private:
|
||||
pid_t pid_ = -1;
|
||||
int port_ = 0;
|
||||
};
|
||||
|
||||
} // namespace vdm::testing
|
||||
|
||||
#endif // VDM_TESTS_NET_TESTSERVER_FIXTURE_HPP
|
||||
@@ -0,0 +1,67 @@
|
||||
# DAEMON → CORE — the engine API `sched/` needs before it can be written
|
||||
|
||||
Status: **resolved**. CORE answered in full: `core/docs/adr-0011-core-response.md`
|
||||
(`lane/core`, commit `7bf5cb5`). Kept as a record of what was asked and the shape of the
|
||||
answer; the API itself is now specified in
|
||||
`docs/adr/0011-admission-control-and-the-segment-budget.md` §"Engine API `sched/` is built
|
||||
against". `sched/` may be written against it.
|
||||
|
||||
Ranking followed `contracts/README.md` rule 4 conventions even though this wasn't a
|
||||
`contracts/` change: new API surface = cheap, land anytime; a behavioural promise (min-1
|
||||
fairness) = needed CORE's explicit sign-off before DAEMON built on the assumption. That
|
||||
sign-off is in.
|
||||
|
||||
---
|
||||
|
||||
## C1. Occupancy read-out, not inference — **resolved**
|
||||
|
||||
Requested `budget()`, `segments_active(TaskId)`, `on_budget_changed`. CORE's answer adds
|
||||
`starved_tasks()` and `starved_since(TaskId)` (amendment A3) and pins two definitions:
|
||||
`segments_active(id)` counts a segment in `connecting` state as held (it is progress, not
|
||||
starvation), and `tasks_starved` counts only `segments_active == 0`. See ADR 0011 §3.6.
|
||||
|
||||
## C2. `set_max_active_segments(uint32_t)` live-apply — **resolved: drain, never kill**
|
||||
|
||||
Confirmed DAEMON's assumption. Lowering runs in-flight segments to their next boundary; no
|
||||
new segment starts while over the new ceiling; nothing is aborted, no partial range lost.
|
||||
If the new ceiling is below the running-task count, CORE honours min-1 for the top-priority
|
||||
subset and reports the rest via `tasks_starved` — DAEMON's governor must reconcile and
|
||||
pause the lowest-priority excess itself (CORE does not auto-pause). See ADR 0011 §2.
|
||||
|
||||
## C3. `set_host_segment_cap(host, uint32_t)` — **resolved, confirmed as proposed**
|
||||
|
||||
CORE keeps the `host → cap` map and derives a task's host from its URL/mirror set; DAEMON
|
||||
owns the table and pushes it. See ADR 0011 §4.
|
||||
|
||||
## C4. Contract gap: `connection.maxActiveSegments` — **resolved by PROTO**
|
||||
|
||||
PROTO landed it (ADR 0012, `connection.maxActiveSegments` default 32,
|
||||
`connection.maxTotalBufferBytes` default 128 MiB, `TaskDetail.effectiveBufferBytes`) while
|
||||
this was in flight. No daemon-local stopgap needed — `sched/` reads the wire field
|
||||
directly. See ADR 0011 §6.
|
||||
|
||||
## C5. Fairness rule sign-off — **resolved, with two amendments**
|
||||
|
||||
CORE confirmed min-1-before-seconds is implementable without a priority-inversion at slot
|
||||
release (two-pass allocator: guarantee pass over zero-slot tasks in DAEMON's priority
|
||||
order, then a growth pass; a released slot always re-enters the pool at pass 1, never
|
||||
handed back locally). Two amendments to what DAEMON assumed:
|
||||
|
||||
- **A1** — "steal" (slot-neutral, unchanged) isn't the whole mechanism; **"yield"** is the
|
||||
slot-transfer operation that actually satisfies min-1 out of a full budget: an
|
||||
over-quota task releases one slot at its next segment boundary, bounded by that
|
||||
segment's remaining bytes.
|
||||
- **A2** — "admission implies progress" is **bounded-delay**, not immediate:
|
||||
`time_to_first_slot ≤ min(next yield boundary, low_speed_secs) + connect_timeout`, not
|
||||
"connect timeout + per-host cap" alone. DAEMON's starvation-invariant assertion window
|
||||
widened from the originally proposed 2 s to `low_speed_secs + connect_timeout` (~45 s)
|
||||
accordingly.
|
||||
|
||||
Priority order (open item 4) is an ordered `TaskId` list pushed via `set_task_order` on
|
||||
change — not an integer, not per-tick. See ADR 0011 §3.
|
||||
|
||||
## C6. Probe pool sized outside the segment budget — **resolved, confirmed**
|
||||
|
||||
Dedicated pool, default size 4, `set_probe_pool_size(uint32_t)`, independent of
|
||||
`maxActiveSegments`; probe cancellation is immediate. DAEMON still bounds probe
|
||||
*submission* on its own side. See ADR 0011 §5.
|
||||
@@ -67,13 +67,23 @@ One file, opened once, `O_WRONLY`. Each segment `pwrite()`s at its own absolute
|
||||
- `posix_fallocate()` the full size up front → contiguous extents, no ENOSPC surprise at
|
||||
99 %, no fragmentation.
|
||||
- Per-segment ring buffer, size = **`buffer_bytes`** (the user-visible "Buffer size"
|
||||
setting). Default 4 MiB, range 64 KiB – 64 MiB. Curl's write callback appends; the
|
||||
buffer is flushed with a single `pwrite` when full or when the segment ends.
|
||||
setting, `connection.bufferBytes` on the wire). Default **1 MiB**, range **64 KiB –
|
||||
16 MiB**. Curl's write callback appends; the buffer is flushed with a single `pwrite`
|
||||
when full or when the segment ends.
|
||||
*This is the single biggest throughput knob and it is exposed in the UI: Options →
|
||||
Downloads → "Write buffer per connection".*
|
||||
- Global cap `max_total_buffer_bytes` (default 256 MiB) so 32 segments × 64 MiB can't OOM
|
||||
the box. The per-segment value is silently reduced to fit and the effective value is
|
||||
reported back to the UI.
|
||||
- Global cap `max_total_buffer_bytes` (`connection.maxTotalBufferBytes`, default
|
||||
**128 MiB**) so a burst of large downloads with a large per-segment buffer can't OOM the
|
||||
box. Combined with `max_active_segments` (`connection.maxActiveSegments`, default
|
||||
**32**) — the ceiling on segments actually transferring at once, across every task, not
|
||||
per download — every live segment's buffer is reduced to fit
|
||||
`max_total_buffer_bytes / live_segment_count` (capped by `max_active_segments`), never
|
||||
below the 64 KiB floor. The requested and effective values are both reported back to
|
||||
the UI (`TaskDetail.bufferBytes` / `.effectiveBufferBytes`) so it can show, for example,
|
||||
"16 MiB (using 4 MiB)". See `docs/adr/0012-buffer-and-segment-budget.md` for the
|
||||
reasoning behind these numbers, including why 4 MiB / 64 MiB / 256 MiB (this section's
|
||||
earlier draft) does not hold ≤ 60 MB RSS once buffers are counted per segment rather
|
||||
than per download.
|
||||
- `posix_fadvise(POSIX_FADV_DONTNEED)` on written ranges — do not let a 40 GB ISO evict
|
||||
the user's entire page cache.
|
||||
- `fdatasync()` on a timer (default 5 s) and on pause, **not** per write.
|
||||
@@ -122,6 +132,6 @@ toggles between Full speed / a saved limit, exactly as IDM does.
|
||||
## 8. Performance targets (M7 gate, `tools/bench/`)
|
||||
|
||||
- Saturate a 1 Gbit link with ≤ 8 % of one core.
|
||||
- ≤ 60 MB RSS with 20 active downloads at default buffers.
|
||||
- ≤ 60 MB RSS with 20 active downloads at default buffers, **given `max_active_segments = 32`** — without that cap, 20 downloads × 8 segments each is 160 live buffers even at the 1 MiB default, and the number does not hold. See `docs/adr/0012-buffer-and-segment-budget.md`.
|
||||
- 10 000-row task list: RPC `download.list` under 50 ms, GUI scroll at 60 fps.
|
||||
- No allocation in the curl write callback hot path (ring buffer is preallocated).
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
# ADR 0011 — Admission control, the segment budget, and who counts what
|
||||
|
||||
**Status:** accepted · **Date:** 2026-09-09 · **Lane:** DAEMON, signed off by CORE
|
||||
**Companion request:** `daemon/docs/core-requests-m1.md` (the engine API this depends on)
|
||||
**CORE's response:** `core/docs/adr-0011-core-response.md` (`lane/core`, commit `7bf5cb5`)
|
||||
— accepted with three amendments (A1–A3, folded in below) and answers to all five open
|
||||
questions. `daemon/src/sched/` is unblocked.
|
||||
|
||||
## Context
|
||||
|
||||
Two lanes were each building a global concurrency governor, neither brief mentioned the
|
||||
other, and they were counting different things.
|
||||
|
||||
* **CORE** added `connection.maxActiveSegments` (default 32, landed on the wire by
|
||||
PROTO's ADR 0012) inside the engine — a ceiling on segments actually transferring at
|
||||
once. It is the mechanism that makes the RSS target in `docs/04` §8 hold
|
||||
(`core/docs/buffer-sizing.md`), so it is not optional.
|
||||
* **DAEMON** is briefed to build "a concurrency governor (global max active, per-queue max,
|
||||
per-host caps)", backed by `connection.maxConcurrentDownloads`, `Queue.maxConcurrent`,
|
||||
schedules and queue order.
|
||||
|
||||
Left alone this lands in one of two states, and both are bad in a way that is hard to
|
||||
diagnose after the fact:
|
||||
|
||||
1. **Double-throttling.** Both lanes enforce a global ceiling, so the effective limit is
|
||||
the minimum of two numbers the user set independently. The link runs at half rate and
|
||||
it reads as a performance bug in the engine, not as a policy collision.
|
||||
2. **The gap.** Each lane assumes the other holds the line. Nobody does, 20 downloads open
|
||||
160 connections, and the RSS budget that `maxActiveSegments` exists to defend is gone.
|
||||
|
||||
There is a third problem underneath both. With `maxActiveSegments = 32` and
|
||||
`connection.maxSegmentsPerDownload` capped at 32, one download can hold the entire segment
|
||||
budget. If the daemon admits a second task and the engine has no slot to give it, the task
|
||||
is *running* and transferring nothing: every DAEMON assumption that admission implies
|
||||
progress — stall detection, speed accounting, queue drain, "when queue completes" — is
|
||||
then wrong. CORE owns that fairness rule. DAEMON cannot write a governor without knowing
|
||||
what it is.
|
||||
|
||||
## Decision
|
||||
|
||||
### 1. Every ceiling is enforced exactly once, by the lane that owns the unit it counts
|
||||
|
||||
This is the whole ADR in one line. The two governors stay two governors, on two axes, with
|
||||
strictly non-overlapping units:
|
||||
|
||||
| Ceiling | Unit | Enforced by | Configured by |
|
||||
|---|---|---|---|
|
||||
| `connection.maxConcurrentDownloads` | tasks | DAEMON | user |
|
||||
| `Queue.maxConcurrent` | tasks | DAEMON | user |
|
||||
| per-host **task** cap | tasks | DAEMON | host table |
|
||||
| schedules, windows, queue order, priority | tasks | DAEMON | user |
|
||||
| `connection.maxActiveSegments` | segments | **CORE** | user |
|
||||
| `connection.maxSegmentsPerDownload` | segments | **CORE** | user, per task |
|
||||
| per-host **segment** cap | segments | **CORE** | host table, pushed by DAEMON |
|
||||
| `bufferBytes` / `maxTotalBufferBytes` | bytes | **CORE** | user |
|
||||
| speed limits (global → queue → task) | bytes/s | **CORE** | user, pushed by DAEMON |
|
||||
|
||||
Corollaries, and these are the parts that actually prevent the two failure modes:
|
||||
|
||||
* **DAEMON never counts segments to make an admission decision.** Not directly, and not by
|
||||
inferring occupancy from a download count. Its governor sees tasks.
|
||||
* **CORE never refuses admission.** `start()` always accepts. The engine paces the task
|
||||
inside the segment budget; it does not decide that the task should not be running. A
|
||||
refusal would be a policy decision, and policy lives in the daemon with the queues and
|
||||
the SQL behind it.
|
||||
* The pattern generalises: **DAEMON decides policy and configures; CORE enforces every
|
||||
ceiling counted in engine-internal units.** Rate limiting (`docs/04` §6) already works
|
||||
this way. Recording it here so it is not re-litigated per subsystem.
|
||||
|
||||
Signed off by CORE as written, including the shared-semaphore rejection in "Alternatives
|
||||
considered" below.
|
||||
|
||||
### 2. The one legitimate coupling — a clamp, not a second enforcement
|
||||
|
||||
DAEMON reads `maxActiveSegments` in exactly one place:
|
||||
|
||||
```
|
||||
effective_max_running_tasks = min(connection.maxConcurrentDownloads,
|
||||
connection.maxActiveSegments)
|
||||
```
|
||||
|
||||
with the same clamp applied per queue against that queue's share. The purpose is narrow:
|
||||
never admit more concurrently-running tasks than the segment budget can give one segment
|
||||
each. It is expressed in tasks, it throttles nothing that CORE also throttles, and it is
|
||||
the reason §3's fairness rule is satisfiable — **CORE's Q1 answer is explicit that min-1
|
||||
liveness depends on DAEMON honouring this clamp.** Admitting 3 running tasks against a
|
||||
budget of 2 starves one by construction and no fairness rule on CORE's side fixes it.
|
||||
|
||||
At the shipped defaults (`maxConcurrentDownloads` 5, `maxActiveSegments` 32) the clamp is
|
||||
not binding. It binds when a user raises concurrency to 64 or lowers the segment budget.
|
||||
|
||||
**After a live lowering of `maxActiveSegments`** below the running-task count, CORE
|
||||
honours min-1 for the top `new_ceiling` tasks in DAEMON's priority order and reports the
|
||||
rest in `tasks_starved` — it does **not** auto-pause them (policy stays with DAEMON). The
|
||||
governor must reconcile its running set against the new clamp on every
|
||||
`on_budget_changed` delivery and pause the lowest-priority excess itself.
|
||||
|
||||
### 3. CORE's fairness rule — what DAEMON is allowed to assume
|
||||
|
||||
CORE owns this; the mechanism below is CORE's, confirmed in its ADR 0011 response.
|
||||
|
||||
1. **Min-1 before seconds.** No task receives a second segment slot while any admitted task
|
||||
holds zero. A task's first slot always outranks another task's growth. Implemented as
|
||||
two ordered passes re-run on every budget-changing edge: a **guarantee pass** over
|
||||
zero-slot tasks in DAEMON's priority order, then a **growth pass** round-robining
|
||||
remaining budget over running tasks up to their effective per-task cap. A released slot
|
||||
always re-enters the pool and re-runs pass 1 from the top — it is never handed back
|
||||
directly to the releasing task — so a task that drops to zero re-enters the guarantee
|
||||
queue at its priority position, not the back. No inversion at slot release.
|
||||
2. Beyond the first slot, remaining budget is distributed round-robin over running tasks in
|
||||
the priority order DAEMON supplies (`set_task_order`), up to each task's effective
|
||||
per-task cap: `min(spec.segments ?? maxSegmentsPerDownload, per-host segment cap, 1 if
|
||||
not resumable)`.
|
||||
3. Slots are released on segment completion, pause, and failure, by two distinct
|
||||
operations:
|
||||
- **Steal** — a worker that finished its range takes the tail of the largest remaining
|
||||
range. Same worker, same slot: slot-neutral, exactly as first described.
|
||||
- **Yield** — the mechanism that actually satisfies rule 1 when the budget is full: the
|
||||
allocator marks an over-quota task to release one slot *at its next segment
|
||||
boundary*, bounded by that segment's remaining bytes. Never a mid-segment kill. This
|
||||
is a slot **transfer**, not slot-neutral — CORE's amendment A1, since "steal" alone
|
||||
doesn't explain how a starved task ever gets its first slot out of a full budget.
|
||||
4. A paused task holds no slots.
|
||||
5. **Admission implies progress, but the bound is delay, not immediacy.** When the budget
|
||||
is full of healthy incumbents, a newly-admitted task's first slot appears at the next
|
||||
yield boundary, hard-capped by the stall timeout:
|
||||
```
|
||||
time_to_first_slot ≤ min(incumbent's next segment boundary, low_speed_secs) + connect_timeout
|
||||
```
|
||||
not "connect timeout + per-host cap" alone, as originally assumed. (CORE amendment A2;
|
||||
a future preemptive-split optimization — truncating an incumbent's range ahead of its
|
||||
current offset — is on the table post-M1 if the yield delay proves painful in soak
|
||||
testing, but doesn't change this API.)
|
||||
6. **Invariant:** `budget().tasks_starved == 0` in steady state, where a task counts as
|
||||
starved only if `segments_active(id) == 0` — a segment in `connecting` state is progress,
|
||||
not starvation (CORE amendment A3). DAEMON asserts this and, if it observes a non-zero
|
||||
value persisting past **`low_speed_secs + connect_timeout` (~45 s, not the originally
|
||||
proposed 2 s** — CORE's A2 correction, since a legitimately full budget with a slow
|
||||
incumbent tail can hold a new task at zero that long with nothing actually wrong), logs
|
||||
a governor-invariant warning and surfaces it in `velox ls --json` using
|
||||
`starved_since(TaskId)` to show how long. It does **not** compensate by throttling
|
||||
admission — compensating is how the two governors would silently grow back into one.
|
||||
|
||||
Consequence for the starvation question in the brief: one download **cannot** permanently
|
||||
take the whole budget away from the next, because rule 1 makes the next task's first slot
|
||||
outrank the incumbent's second via yield. A single download alone in the system does
|
||||
legitimately grow to 32 segments, and gives slots back (bounded-delay, per rule 5) as
|
||||
tasks arrive — growth is opportunistic, the first slot is guaranteed within a bounded time.
|
||||
|
||||
### 4. Per-host caps are split by unit, from one table
|
||||
|
||||
Both briefs say "per-host caps" and they are not the same cap.
|
||||
|
||||
* CORE enforces per-host **segment** caps — it owns the connections and is the only place
|
||||
segments are counted (`docs/04` §3, "clamped per-host by settings"). CORE derives a
|
||||
task's host from its URL and mirror set; DAEMON does not need to push per-task host
|
||||
resolution, only the cap table.
|
||||
* DAEMON enforces a per-host **task** cap, set to that host's segment cap, so it can never
|
||||
admit more tasks for one host than that host can be given one segment each. Without this,
|
||||
rule 3.1 is unsatisfiable: four tasks on a host capped at 4 connections is fine, five is
|
||||
a guaranteed starved task no fairness rule can fix.
|
||||
* The table itself is DAEMON state (SQLite, `settings`), pushed into the engine via
|
||||
`set_host_segment_cap(std::string host, uint32_t)`. One source of truth, two enforcement
|
||||
points, different units.
|
||||
|
||||
### 5. Probes do not consume segment slots
|
||||
|
||||
A `download.probe` is a HEAD or a one-byte ranged GET. Charging it against the segment
|
||||
budget would let a burst of probes starve transfers, and probes are on the latency path for
|
||||
`capture.offer`'s 750 ms deadline. CORE bounds concurrent probes with its own dedicated
|
||||
pool, default size 4, `set_probe_pool_size(uint32_t)`, entirely independent of the segment
|
||||
budget — confirmed by CORE. Probe cancellation is immediate, so `capture.offer` can answer
|
||||
`ignore` first and probe after with no risk of blocking on a stuck probe. CORE bounds probe
|
||||
*concurrency*; DAEMON still bounds probe *submission* on its own side (queue depth is a
|
||||
DAEMON policy question, not an engine one).
|
||||
|
||||
### 6. `connection.maxActiveSegments` is now on the wire (PROTO ADR 0012)
|
||||
|
||||
Resolved: PROTO landed `connection.maxActiveSegments` (default 32) and
|
||||
`connection.maxTotalBufferBytes` (default 128 MiB) in `Settings.schema.json`/`SettingKey`
|
||||
in ADR 0012, alongside `TaskDetail.effectiveBufferBytes`. DAEMON reads and writes it
|
||||
through `settings.get`/`settings.set` like any other connection setting — no daemon-local
|
||||
stopgap needed. `sched/` can reference the wire field directly.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
**DAEMON enforces both.** The governor would have to predict each task's effective segment
|
||||
count to spend a segment budget in task units — but that count depends on the probe result,
|
||||
the per-host cap, the resumability demotion and live steals, all engine-internal and all
|
||||
changing continuously. Predicting it means either over-admitting (the gap) or leaving the
|
||||
link idle. Rejected: it asks the daemon to model the engine.
|
||||
|
||||
**CORE enforces both.** The engine would take every task and decide which run. That drags
|
||||
queues, schedules, priority, and "when queue completes" into `core/`, which the layering
|
||||
rule forbids and which would need SQL to be correct. Rejected.
|
||||
|
||||
**A shared semaphore object handed to both lanes.** Superficially the "one counter" answer,
|
||||
but it makes a mutable engine resource part of the daemon's API surface, inverts the
|
||||
dependency direction, and is the first thing that will deadlock under pause-during-steal.
|
||||
Rejected: one counter, one owner, read-only snapshots for everyone else.
|
||||
|
||||
## Consequences
|
||||
|
||||
* `daemon/src/sched/` may be written against a task-unit model only. A segment count
|
||||
appearing in an admission decision is a review-blocking defect.
|
||||
* DAEMON reads occupancy through the engine API below rather than inferring it —
|
||||
needed to project `TaskSummary.segments` (ADR 0010: the *effective* count) without
|
||||
guessing.
|
||||
* CORE's fairness rule has a test DAEMON can point at: N tasks admitted, budget smaller
|
||||
than N × their per-task caps, assert every task reaches ≥ 1 segment within
|
||||
`low_speed_secs + connect_timeout`, and `tasks_starved == 0` in steady state thereafter.
|
||||
* The governor must reconcile its running set on every `on_budget_changed` delivery,
|
||||
pausing the lowest-priority excess when a live lowering of `maxActiveSegments` leaves
|
||||
some running tasks permanently below `new_ceiling` (§2).
|
||||
|
||||
## Engine API `sched/` is built against (CORE, `core/docs/adr-0011-core-response.md`)
|
||||
|
||||
```
|
||||
void set_max_active_segments(uint32_t); // drain-not-kill (§2, §3.5)
|
||||
void set_host_segment_cap(std::string host, uint32_t); // §4
|
||||
void set_task_order(std::span<const TaskId>); // pushed on change, not per tick
|
||||
void set_probe_pool_size(uint32_t); // default 4, §5
|
||||
|
||||
struct EngineBudget { uint32_t total; uint32_t active; uint32_t tasks_starved; };
|
||||
EngineBudget budget() const;
|
||||
uint32_t segments_active(TaskId) const; // includes `connecting`
|
||||
std::vector<TaskId> starved_tasks() const;
|
||||
std::optional<SteadyTime> starved_since(TaskId) const;
|
||||
void on_budget_changed(std::function<void(EngineBudget)>); // 4 Hz + immediate on the
|
||||
// tasks_starved 0↔nonzero edge
|
||||
```
|
||||
|
||||
All of it lands with CORE's stage 6 (segmenter/stealer) / stage 8 (download_task) — not on
|
||||
the M1 critical path ahead of where `sched/` needs it, per CORE.
|
||||
|
||||
## Resolved questions (were open, now answered by CORE)
|
||||
|
||||
1. **Min-1 without priority inversion at slot release — yes**, per §3.1 above.
|
||||
2. **Probe pool outside the segment budget, size 4 — confirmed**, §5.
|
||||
3. **Live `set_max_active_segments()` — drain, never kill**, §2/§3.5.
|
||||
4. **Priority shape — an ordered `TaskId` list, pushed on change**, not an integer and not
|
||||
per-tick. DAEMON already owns the total order (queue precedence, admission-time
|
||||
tie-break); pushing an integer would force CORE to reimplement tie-breaking, which is
|
||||
DAEMON policy.
|
||||
5. **Budget-change callback — 4 Hz coalesced, plus an immediate fire on the
|
||||
`tasks_starved` 0↔non-zero transition** so the steady-state invariant check and any UI
|
||||
reaction aren't lagged by up to 250 ms.
|
||||
@@ -0,0 +1,106 @@
|
||||
# ADR 0012 — Buffer bounds, the total-buffer cap, and `maxActiveSegments`
|
||||
|
||||
**Status:** accepted · **Date:** 2026-09-09 · **Lane:** PROTO
|
||||
**Answers:** `core/docs/buffer-sizing.md` (CORE's B4)
|
||||
|
||||
## Context
|
||||
|
||||
`docs/04` §4 (line 70) said 64 KiB – 64 MiB, default 4 MiB, global cap 256 MiB. The frozen
|
||||
contract said 4 KiB – 8 MiB in four places. Neither matched the other, and CORE's
|
||||
reconciliation found the deeper problem: `docs/04`'s own performance target — "≤ 60 MB RSS
|
||||
with 20 active downloads at default buffers" (§8, line 125) — assumed one buffer per
|
||||
*download*. The design is one buffer per *segment* ("Per-segment ring buffer", §4 line 68).
|
||||
Twenty downloads at the 8-segment default is 160 buffers, not 20. At any of the candidate
|
||||
defaults (4 MiB, 2 MiB, even 1 MiB) that arithmetic busts the 60 MB target by 3–4×, and the
|
||||
256 MiB global cap does too on its own.
|
||||
|
||||
CORE also found a second wire gap: the clamp the engine has to implement — reduce every
|
||||
live segment's buffer to fit the global cap — has no field to report the reduced value on,
|
||||
so the GUI cannot show what a download is actually using, only what was requested.
|
||||
|
||||
## Decision
|
||||
|
||||
**Bounds**, in all four schema locations (`DownloadSpec.bufferBytes`,
|
||||
`TaskDetail.bufferBytes`, `download.update`'s patch, `Settings.connection.bufferBytes`):
|
||||
|
||||
| | Old (frozen 1.0.0) | New (1.1.0) |
|
||||
|---|---|---|
|
||||
| minimum | 4096 (4 KiB) | **65536 (64 KiB)** |
|
||||
| maximum | 8388608 (8 MiB) | **16777216 (16 MiB)** |
|
||||
| default | *(unstated on the wire)* | **1048576 (1 MiB)** |
|
||||
|
||||
64 KiB because 4 KiB is smaller than one libcurl HTTP/2 write-callback delivery — a buffer
|
||||
that small does not coalesce anything, it just adds a layer between curl and `pwrite` that
|
||||
does nothing. 16 MiB because throughput from write size is flat past ~1–4 MiB on NVMe; the
|
||||
only thing 8–16 MiB buys past that is absorbing a disk stall without stalling the socket,
|
||||
and past 16 MiB there is stall-cover left to buy but not memory left to spend it on. 1 MiB
|
||||
default because it is the only default of the three considered (4 MiB, 2 MiB, 1 MiB) for
|
||||
which the RSS target below actually holds.
|
||||
|
||||
**Two new settings keys**, `connection.maxTotalBufferBytes` (default 134217728, 128 MiB)
|
||||
and `connection.maxActiveSegments` (default 32). Both are new, both minor. Without the
|
||||
first, CORE's clamp has no configuration surface. Without the second, "20 active downloads"
|
||||
has no wire meaning distinct from "160 live TLS connections", which is the actual RSS
|
||||
driver — `docs/01` §2 already assumes a cap ("~8 active segments" per transfer thread) that
|
||||
`docs/04` never stated as a number.
|
||||
|
||||
**`TaskDetail.effectiveBufferBytes`** (new, per B2a, already accepted in
|
||||
`contracts/proto-answers-m1.md`): what a live segment is actually using, after the daemon
|
||||
divides `maxTotalBufferBytes` across live segments (capped at `maxActiveSegments`) and
|
||||
clamps down to fit. `bufferBytes` stays the requested value on every type; effective and
|
||||
requested sit side by side wherever both apply, the same pattern `TaskSummary.segments` /
|
||||
`DownloadSpec.segments` already established in ADR 0010.
|
||||
|
||||
**The RSS target: 60 MB stands, and is now conditional in writing.** CORE offered a choice
|
||||
— keep 60 MB and rely on `maxActiveSegments = 32`, or raise it to 120 MB and use a looser
|
||||
cap. This ADR keeps 60 MB, because "lean daemon" is already a stated goal (`docs/01`) and
|
||||
CORE's own arithmetic gets to 45–50 MB at the chosen defaults — comfortable margin, not a
|
||||
number that only barely holds. `docs/04` §8 now states the RSS target is conditional on
|
||||
`maxActiveSegments = 32` and default buffers, so it stops being a claim nobody can check.
|
||||
|
||||
## Why this is a minor bump, not major
|
||||
|
||||
Every change either widens a range (4 KiB–8 MiB → 64 KiB–16 MiB, which is not a superset in
|
||||
both directions — see below), adds a field, or adds a settings key. Rule 4 in
|
||||
`contracts/README.md`: optional field or new method is minor; only rename/remove/retype is
|
||||
major.
|
||||
|
||||
**The one place this needs a caller's attention despite being "minor":** the new minimum
|
||||
(64 KiB) is *higher* than the old one (4 KiB), and the new maximum (16 MiB) is *lower* than
|
||||
old `docs/04`'s stated 64 MiB (though *higher* than the old frozen 8 MiB). A value that
|
||||
validated against the pre-1.1.0 schema — say, `bufferBytes: 2048` — no longer validates.
|
||||
Nothing shipped against 1.0.0 yet (this repository is the only consumer), so there is no
|
||||
live client to break, but the general rule for a future minor bump that narrows one bound
|
||||
while widening another is: **check it against every existing fixture before shipping**,
|
||||
which `tests/conformance/check_contract.py` now does mechanically — this bump passed
|
||||
because both prior fixture values (4 MiB) happened to fall inside the new range too.
|
||||
|
||||
## Consequences
|
||||
|
||||
* `TaskDetail` gains a field; `Settings` and `SettingKey` gain two keys; four
|
||||
`bufferBytes` constraint blocks change bounds and gain a stated default.
|
||||
* DAEMON's scheduler needs `maxActiveSegments` to decide what to admit — this is the
|
||||
concrete data DAEMON asked for in its own admission-control proposal (currently under
|
||||
discussion as a separate ADR against a different number; PROTO takes no position on that
|
||||
split here beyond supplying the wire field DAEMON needs to read).
|
||||
* `docs/04` §4 and §8 are updated in the same change, per `CLAUDE.md` rule 5 (docs move
|
||||
with the behaviour they describe) and per CORE's explicit request to land schema, docs
|
||||
and ADR together.
|
||||
* The `download.get` fixture now shows a real clamp — 16 MiB requested, 4 MiB effective —
|
||||
rather than a case where the cap happens not to bind, so the requested/effective split
|
||||
actually gets exercised by conformance.
|
||||
|
||||
## Alternatives rejected
|
||||
|
||||
**120 MB RSS target, looser cap.** CORE offered this and it is defensible — 32 live TLS
|
||||
connections have an irreducible cost, and IDM budgets more. Rejected because the tighter
|
||||
number is achievable with margin per CORE's own arithmetic and "lean daemon" is already a
|
||||
stated design goal; raising a target because it is easier is the wrong direction to move
|
||||
without a measurement forcing it. If M7's `tools/bench` run shows 60 MB is not actually
|
||||
reachable in practice, that is new information and reopens this ADR — it is not a reason to
|
||||
soften the number pre-emptively.
|
||||
|
||||
**Keeping `docs/04`'s 64 MiB ceiling.** Rejected on CORE's own reachability argument:
|
||||
against a 128 MiB total cap, 64 MiB per segment is unreachable past 2 live segments, which
|
||||
under correct per-segment accounting is a quarter of one default 8-segment download. A
|
||||
ceiling nothing can reach is not a ceiling, it is dead text.
|
||||
@@ -3,7 +3,7 @@
|
||||
//
|
||||
// Source: contracts/schema/**
|
||||
// Generator: contracts/codegen/gen_ts.py
|
||||
// Contract: v1.0.0
|
||||
// Contract: v1.1.0
|
||||
//
|
||||
// Hand-editing this file is a merge blocker. Fix the schema and regenerate:
|
||||
// python3 contracts/codegen/gen_ts.py
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//
|
||||
// Source: contracts/schema/**
|
||||
// Generator: contracts/codegen/gen_ts.py
|
||||
// Contract: v1.0.0
|
||||
// Contract: v1.1.0
|
||||
//
|
||||
// Hand-editing this file is a merge blocker. Fix the schema and regenerate:
|
||||
// python3 contracts/codegen/gen_ts.py
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//
|
||||
// Source: contracts/schema/**
|
||||
// Generator: contracts/codegen/gen_ts.py
|
||||
// Contract: v1.0.0
|
||||
// Contract: v1.1.0
|
||||
//
|
||||
// Hand-editing this file is a merge blocker. Fix the schema and regenerate:
|
||||
// python3 contracts/codegen/gen_ts.py
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//
|
||||
// Source: contracts/schema/**
|
||||
// Generator: contracts/codegen/gen_ts.py
|
||||
// Contract: v1.0.0
|
||||
// Contract: v1.1.0
|
||||
//
|
||||
// Hand-editing this file is a merge blocker. Fix the schema and regenerate:
|
||||
// python3 contracts/codegen/gen_ts.py
|
||||
@@ -11,7 +11,7 @@
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
export const PROTOCOL_VERSION = "1.0.0";
|
||||
export const PROTOCOL_VERSION = "1.1.0";
|
||||
|
||||
/**
|
||||
* Every error code the daemon may return. Adding one is a minor bump; changing the meaning
|
||||
@@ -214,6 +214,12 @@ export interface DownloadSpec {
|
||||
* connection.maxSegmentsPerDownload.
|
||||
*/
|
||||
segments?: number | null;
|
||||
/**
|
||||
* Requested write buffer per segment, in bytes. null means use connection.bufferBytes.
|
||||
* Default 1 MiB; range 64 KiB - 16 MiB. Silently reduced to fit
|
||||
* connection.maxTotalBufferBytes across all live segments; the effective value is reported
|
||||
* back as TaskDetail.effectiveBufferBytes.
|
||||
*/
|
||||
bufferBytes?: number | null;
|
||||
startMode?: StartMode;
|
||||
description?: string | null;
|
||||
@@ -440,7 +446,7 @@ export interface Segment {
|
||||
* must not invent a key that is not here. Kept in lockstep with Settings.schema.json by a
|
||||
* conformance check.
|
||||
*/
|
||||
export type SettingKey = "general.launchOnLogin" | "general.minimizeToTray" | "general.showDropTarget" | "general.confirmOnExit" | "general.language" | "general.checkForUpdates" | "capture.enabled" | "capture.monitoredExtensions" | "capture.monitoredMimeTypes" | "capture.minSizeBytes" | "capture.excludedHosts" | "capture.bypassModifier" | "capture.autoStartTypes" | "saveTo.defaultDir" | "saveTo.tempDir" | "saveTo.allowedRoots" | "saveTo.fileExistsPolicy" | "saveTo.createSubfolderPerSite" | "connection.preset" | "connection.maxSegmentsPerDownload" | "connection.bufferBytes" | "connection.maxConcurrentDownloads" | "connection.timeoutSec" | "connection.maxRetries" | "connection.retryBackoffSec" | "downloads.speedLimitBps" | "downloads.speedLimitEnabled" | "downloads.virusScanCommand" | "downloads.postDownloadCommand" | "downloads.duplicatePolicy" | "downloads.verifyChecksums" | "proxy.mode" | "proxy.host" | "proxy.port" | "proxy.username" | "proxy.bypassHosts" | "proxy.pacUrl" | "sounds.enabled" | "sounds.onComplete" | "sounds.onQueueComplete" | "sounds.onError";
|
||||
export type SettingKey = "general.launchOnLogin" | "general.minimizeToTray" | "general.showDropTarget" | "general.confirmOnExit" | "general.language" | "general.checkForUpdates" | "capture.enabled" | "capture.monitoredExtensions" | "capture.monitoredMimeTypes" | "capture.minSizeBytes" | "capture.excludedHosts" | "capture.bypassModifier" | "capture.autoStartTypes" | "saveTo.defaultDir" | "saveTo.tempDir" | "saveTo.allowedRoots" | "saveTo.fileExistsPolicy" | "saveTo.createSubfolderPerSite" | "connection.preset" | "connection.maxSegmentsPerDownload" | "connection.bufferBytes" | "connection.maxTotalBufferBytes" | "connection.maxActiveSegments" | "connection.maxConcurrentDownloads" | "connection.timeoutSec" | "connection.maxRetries" | "connection.retryBackoffSec" | "downloads.speedLimitBps" | "downloads.speedLimitEnabled" | "downloads.virusScanCommand" | "downloads.postDownloadCommand" | "downloads.duplicatePolicy" | "downloads.verifyChecksums" | "proxy.mode" | "proxy.host" | "proxy.port" | "proxy.username" | "proxy.bypassHosts" | "proxy.pacUrl" | "sounds.enabled" | "sounds.onComplete" | "sounds.onQueueComplete" | "sounds.onError";
|
||||
export const SETTING_KEY_VALUES = [
|
||||
"general.launchOnLogin",
|
||||
"general.minimizeToTray",
|
||||
@@ -463,6 +469,8 @@ export const SETTING_KEY_VALUES = [
|
||||
"connection.preset",
|
||||
"connection.maxSegmentsPerDownload",
|
||||
"connection.bufferBytes",
|
||||
"connection.maxTotalBufferBytes",
|
||||
"connection.maxActiveSegments",
|
||||
"connection.maxConcurrentDownloads",
|
||||
"connection.timeoutSec",
|
||||
"connection.maxRetries",
|
||||
@@ -553,6 +561,11 @@ export interface Settings {
|
||||
"saveTo.createSubfolderPerSite"?: boolean;
|
||||
"connection.preset"?: SettingsConnectionPreset;
|
||||
"connection.maxSegmentsPerDownload"?: number;
|
||||
/**
|
||||
* Default per-segment write buffer, in bytes, when a task does not request its own.
|
||||
* Default 1 MiB (1048576); range 64 KiB - 16 MiB. This is the single biggest throughput
|
||||
* knob and is exposed in Options -> Downloads -> 'Write buffer per connection'.
|
||||
*/
|
||||
"connection.bufferBytes"?: number;
|
||||
"connection.maxConcurrentDownloads"?: number;
|
||||
"connection.timeoutSec"?: number;
|
||||
@@ -574,6 +587,21 @@ export interface Settings {
|
||||
"sounds.onComplete"?: string;
|
||||
"sounds.onQueueComplete"?: string;
|
||||
"sounds.onError"?: string;
|
||||
/**
|
||||
* Global cap on write-buffer memory across every live segment, in bytes. Default 128 MiB
|
||||
* (134217728). Every live segment's buffer is reduced to fit maxTotalBufferBytes / (live
|
||||
* segment count, capped at maxActiveSegments); the reduced value is reported per task as
|
||||
* TaskDetail.effectiveBufferBytes. Exists so a burst of large downloads with a large
|
||||
* per-segment buffer cannot exhaust memory.
|
||||
*/
|
||||
"connection.maxTotalBufferBytes"?: number;
|
||||
/**
|
||||
* Global ceiling on segments actually transferring at once, across every task. Default 32.
|
||||
* This is the real bound behind '20 active downloads': the rest of each download's
|
||||
* segments queue rather than all dialling out simultaneously. DAEMON's scheduler needs
|
||||
* this value to decide what to admit; CORE enforces it.
|
||||
*/
|
||||
"connection.maxActiveSegments"?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -708,7 +736,19 @@ export interface TaskDetail {
|
||||
referrer?: string | null;
|
||||
userAgent?: string | null;
|
||||
mime?: string | null;
|
||||
/**
|
||||
* The REQUESTED write buffer per segment. See effectiveBufferBytes for what is actually in
|
||||
* use.
|
||||
*/
|
||||
bufferBytes?: number | null;
|
||||
/**
|
||||
* The write buffer actually in use per live segment, right now. May be well below
|
||||
* bufferBytes: the daemon reduces every live segment's buffer to fit
|
||||
* connection.maxTotalBufferBytes across connection.maxActiveSegments
|
||||
* concurrently-transferring segments, and reports the reduced value here so the GUI can
|
||||
* show '16 MiB (using 4 MiB)'. null before the task has started its first segment.
|
||||
*/
|
||||
effectiveBufferBytes?: number | null;
|
||||
/** Absolute path of the .veloxpart file while the task is unfinished. */
|
||||
partPath?: string | null;
|
||||
checksum?: Checksum | null;
|
||||
@@ -995,6 +1035,11 @@ export interface DownloadUpdateParamsPatch {
|
||||
* re-segmented underneath the user.
|
||||
*/
|
||||
segments?: number | null;
|
||||
/**
|
||||
* The REQUESTED write buffer per segment. Subject to the same maxTotalBufferBytes
|
||||
* reduction as DownloadSpec.bufferBytes; the effective value comes back on the next
|
||||
* download.get.
|
||||
*/
|
||||
bufferBytes?: number | null;
|
||||
checksum?: Checksum | null;
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//
|
||||
// Source: contracts/schema/**
|
||||
// Generator: contracts/codegen/gen_ts.py
|
||||
// Contract: v1.0.0
|
||||
// Contract: v1.1.0
|
||||
//
|
||||
// Hand-editing this file is a merge blocker. Fix the schema and regenerate:
|
||||
// python3 contracts/codegen/gen_ts.py
|
||||
@@ -490,7 +490,7 @@ export function validateDownloadSpec(v: unknown, path = ''): Validated<DownloadS
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "segments", path, vLimited(vInteger, { minimum: 1, maximum: 32 }), out);
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "bufferBytes", path, vLimited(vInteger, { minimum: 4096, maximum: 8388608 }), out);
|
||||
r = opt(v, "bufferBytes", path, vLimited(vInteger, { minimum: 65536, maximum: 16777216 }), out);
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "startMode", path, validateStartMode, out);
|
||||
if (!r.ok) return r;
|
||||
@@ -754,7 +754,7 @@ export function validateSettings(v: unknown, path = ''): Validated<Settings> {
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "connection.maxSegmentsPerDownload", path, vLimited(vInteger, { minimum: 1, maximum: 32 }), out);
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "connection.bufferBytes", path, vLimited(vInteger, { minimum: 4096, maximum: 8388608 }), out);
|
||||
r = opt(v, "connection.bufferBytes", path, vLimited(vInteger, { minimum: 65536, maximum: 16777216 }), out);
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "connection.maxConcurrentDownloads", path, vLimited(vInteger, { minimum: 1, maximum: 64 }), out);
|
||||
if (!r.ok) return r;
|
||||
@@ -796,6 +796,10 @@ export function validateSettings(v: unknown, path = ''): Validated<Settings> {
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "sounds.onError", path, vString, out);
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "connection.maxTotalBufferBytes", path, vLimited(vInteger, { minimum: 16777216, maximum: 2147483648 }), out);
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "connection.maxActiveSegments", path, vLimited(vInteger, { minimum: 1, maximum: 256 }), out);
|
||||
if (!r.ok) return r;
|
||||
return { ok: true, value: out as unknown as Settings };
|
||||
}
|
||||
|
||||
@@ -824,7 +828,9 @@ export function validateTaskDetail(v: unknown, path = ''): Validated<TaskDetail>
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "mime", path, vString, out);
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "bufferBytes", path, vLimited(vInteger, { minimum: 4096, maximum: 8388608 }), out);
|
||||
r = opt(v, "bufferBytes", path, vLimited(vInteger, { minimum: 65536, maximum: 16777216 }), out);
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "effectiveBufferBytes", path, vLimited(vInteger, { minimum: 65536, maximum: 16777216 }), out);
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "partPath", path, vString, out);
|
||||
if (!r.ok) return r;
|
||||
@@ -1336,7 +1342,7 @@ export function validateDownloadUpdateParamsPatch(v: unknown, path = ''): Valida
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "segments", path, vLimited(vInteger, { minimum: 1, maximum: 32 }), out);
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "bufferBytes", path, vLimited(vInteger, { minimum: 4096, maximum: 8388608 }), out);
|
||||
r = opt(v, "bufferBytes", path, vLimited(vInteger, { minimum: 65536, maximum: 16777216 }), out);
|
||||
if (!r.ok) return r;
|
||||
r = opt(v, "checksum", path, validateChecksum, out);
|
||||
if (!r.ok) return r;
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//
|
||||
// Source: contracts/schema/**
|
||||
// Generator: contracts/codegen/gen_cpp.py
|
||||
// Contract: v1.0.0
|
||||
// Contract: v1.1.0
|
||||
//
|
||||
// Hand-editing this file is a merge blocker. Fix the schema and regenerate:
|
||||
// python3 contracts/codegen/gen_cpp.py
|
||||
|
||||
Reference in New Issue
Block a user