core: net/http_client — libcurl multi wrapper (stage 2)
One HttpClient owns a small pool of workers, each with its own CURLM; an easy handle lives on one worker for its life. Public start/pause/resume/ cancel enqueue a command + curl_multi_wakeup(); callbacks (on_head / on_data / on_finished) run on the worker thread and return a DataAction (proceed / pause / abort). Covers redirects (final-response head only), ranges (inclusive ByteRange -> CURLOPT_RANGE), proxy/SOCKS5, basic/digest auth, cookies, verbatim headers, stall detection, a coarse recv-rate cap, and a curl_share DNS/TLS cache across workers. CURLcode + HTTP status -> vdm::Error in net/curl_error. A probe is on_head returning abort: it finishes successfully (head_complete), not canceled. Tests drive tools/testserver: full GET, ranged 206, redirect chain, 404 -> not_found, connection refused -> connect_failed, HEAD probe + ranged 0-0 probe (no body), cancel mid-transfer, pause/resume completes. Skip cleanly if testserver isn't in the tree. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01HPPSGhiArbvQgwC2DNiURS
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,125 @@
|
||||
// vdm/net/http_client.hpp — a libcurl-multi wrapper for the download engine.
|
||||
//
|
||||
// One HttpClient owns a small pool of worker threads, each with its own curl_multi
|
||||
// (curl handles are not thread-safe; a handle lives on exactly one worker for its life).
|
||||
// The engine hands it a Request plus callbacks and gets back a Transfer handle it can
|
||||
// pause / resume / cancel from any thread.
|
||||
//
|
||||
// Layering: this is the ONLY core header that pulls in libcurl, and only in its .cpp —
|
||||
// nothing here exposes a curl type. No JSON / SQL / Qt / RPC (CLAUDE.md §3).
|
||||
//
|
||||
// Callbacks run ON THE WORKER THREAD, one transfer at a time for a given Transfer.
|
||||
// They must not block (that stalls every other transfer on that worker) and must not
|
||||
// call back into this Transfer's pause/resume/cancel re-entrantly — post that work
|
||||
// elsewhere. on_data must not allocate on the hot path (AGENT-CORE); the ring buffer it
|
||||
// writes into is preallocated by the caller (stage 4).
|
||||
//
|
||||
// This header compiles standalone.
|
||||
|
||||
#ifndef VDM_NET_HTTP_CLIENT_HPP
|
||||
#define VDM_NET_HTTP_CLIENT_HPP
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "vdm/net/http_types.hpp"
|
||||
#include "vdm/util/bytes.hpp"
|
||||
#include "vdm/util/result.hpp"
|
||||
|
||||
namespace vdm::net {
|
||||
|
||||
// What on_data tells the client to do with the transfer after this chunk.
|
||||
enum class DataAction {
|
||||
proceed, // keep receiving
|
||||
pause, // stop receiving; resumes on Transfer::resume()
|
||||
abort, // end the transfer now (finishes with Error::canceled)
|
||||
};
|
||||
|
||||
struct TransferStats {
|
||||
std::uint64_t bytes_received = 0;
|
||||
long http_status = 0;
|
||||
std::string effective_url;
|
||||
// Timings in milliseconds (from CURLINFO_*_TIME_T), 0 if the phase didn't happen.
|
||||
long namelookup_ms = 0;
|
||||
long connect_ms = 0;
|
||||
long appconnect_ms = 0; // TLS handshake done
|
||||
long starttransfer_ms = 0; // first response byte
|
||||
long total_ms = 0;
|
||||
};
|
||||
|
||||
struct TransferCallbacks {
|
||||
// Response headers are in. Called at most once per transfer (a followed redirect's
|
||||
// intermediate headers are not delivered). If the caller only wanted headers (a
|
||||
// probe), return DataAction::abort here.
|
||||
std::function<DataAction(const ResponseHead &)> on_head;
|
||||
|
||||
// A chunk of body bytes. The span is valid only for the duration of the call.
|
||||
std::function<DataAction(ConstByteSpan)> on_data;
|
||||
|
||||
// The transfer ended — success carries stats, failure carries the mapped error
|
||||
// (ErrorInfo.http_status is set for HTTP-status failures). Always called exactly once,
|
||||
// last.
|
||||
std::function<void(Result<TransferStats>)> on_finished;
|
||||
};
|
||||
|
||||
class HttpClient;
|
||||
|
||||
// Lightweight handle to a running transfer. Copyable (shared state). All methods are
|
||||
// safe to call from any thread; they post to the owning worker and return immediately.
|
||||
// Dropping the last handle does NOT cancel — call cancel() for that.
|
||||
class Transfer {
|
||||
public:
|
||||
Transfer() = default;
|
||||
|
||||
[[nodiscard]] std::uint64_t id() const noexcept;
|
||||
[[nodiscard]] bool valid() const noexcept { return static_cast<bool>(state_); }
|
||||
|
||||
void pause(); // no-op if already paused / finished
|
||||
void resume(); // no-op if not paused / finished
|
||||
void cancel(); // idempotent; on_finished fires with Error::canceled
|
||||
|
||||
private:
|
||||
friend class HttpClient;
|
||||
struct State;
|
||||
explicit Transfer(std::shared_ptr<State> s) : state_(std::move(s)) {}
|
||||
std::shared_ptr<State> state_;
|
||||
};
|
||||
|
||||
class HttpClient {
|
||||
public:
|
||||
struct Options {
|
||||
// Worker threads, each with its own curl_multi. New transfers are assigned
|
||||
// round-robin. 0 => pick from hardware_concurrency (min 1, max 4).
|
||||
unsigned workers = 0;
|
||||
// CURLMOPT_MAX_TOTAL_CONNECTIONS per worker (0 = curl default).
|
||||
long max_connections_per_worker = 0;
|
||||
// Shared DNS + TLS-session cache across this client's workers (curl_share).
|
||||
bool share_dns_and_tls = true;
|
||||
};
|
||||
|
||||
HttpClient(); // default Options
|
||||
explicit HttpClient(Options opts);
|
||||
~HttpClient();
|
||||
|
||||
HttpClient(const HttpClient &) = delete;
|
||||
HttpClient &operator=(const HttpClient &) = delete;
|
||||
|
||||
[[nodiscard]] unsigned worker_count() const noexcept;
|
||||
|
||||
// Start a transfer. The Request is consumed (moved). Returns an invalid Transfer and
|
||||
// never calls the callbacks only if the client is shutting down; every other failure
|
||||
// (bad URL, DNS, ...) is delivered through on_finished.
|
||||
Transfer start(Request req, TransferCallbacks cbs);
|
||||
|
||||
private:
|
||||
friend class Transfer; // Transfer posts pause/resume/cancel commands to Impl
|
||||
struct Impl;
|
||||
std::unique_ptr<Impl> impl_;
|
||||
};
|
||||
|
||||
} // namespace vdm::net
|
||||
|
||||
#endif // VDM_NET_HTTP_CLIENT_HPP
|
||||
@@ -0,0 +1,156 @@
|
||||
// vdm/net/http_types.hpp — value types for HTTP requests and responses.
|
||||
//
|
||||
// No libcurl in this header: it is the vocabulary the rest of core/ speaks to the net
|
||||
// layer. http_client.hpp is the only place curl leaks in, and only in its .cpp.
|
||||
//
|
||||
// This header compiles standalone.
|
||||
|
||||
#ifndef VDM_NET_HTTP_TYPES_HPP
|
||||
#define VDM_NET_HTTP_TYPES_HPP
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace vdm::net {
|
||||
|
||||
enum class Method { get, head };
|
||||
|
||||
[[nodiscard]] constexpr std::string_view method_name(Method m) noexcept {
|
||||
return m == Method::head ? "HEAD" : "GET";
|
||||
}
|
||||
|
||||
// A closed byte range [first, last]. `last == kUnbounded` means "to the end of the
|
||||
// resource" (`Range: bytes=first-`). This mirrors HTTP Range semantics — inclusive on
|
||||
// both ends — deliberately: see core/docs/proto-requests-m1.md B3.
|
||||
struct ByteRange {
|
||||
static constexpr std::uint64_t kUnbounded = ~std::uint64_t{0};
|
||||
|
||||
std::uint64_t first = 0;
|
||||
std::uint64_t last = kUnbounded;
|
||||
|
||||
[[nodiscard]] bool bounded() const noexcept { return last != kUnbounded; }
|
||||
[[nodiscard]] std::optional<std::uint64_t> length() const noexcept {
|
||||
if (!bounded())
|
||||
return std::nullopt;
|
||||
return last - first + 1;
|
||||
}
|
||||
// "bytes=100-199" or "bytes=100-"
|
||||
[[nodiscard]] std::string to_header_value() const {
|
||||
std::string v = "bytes=";
|
||||
v += std::to_string(first);
|
||||
v += '-';
|
||||
if (bounded())
|
||||
v += std::to_string(last);
|
||||
return v;
|
||||
}
|
||||
};
|
||||
|
||||
struct HeaderField {
|
||||
std::string name;
|
||||
std::string value;
|
||||
};
|
||||
|
||||
// Case-insensitive view over response headers. Not a multimap for perf — the header set
|
||||
// on a download response is tiny; linear scan is fine and keeps this allocation-light.
|
||||
class HeaderList {
|
||||
public:
|
||||
void add(std::string name, std::string value) {
|
||||
fields_.push_back({std::move(name), std::move(value)});
|
||||
}
|
||||
void clear() noexcept { fields_.clear(); }
|
||||
|
||||
[[nodiscard]] const std::vector<HeaderField> &fields() const noexcept { return fields_; }
|
||||
[[nodiscard]] bool empty() const noexcept { return fields_.empty(); }
|
||||
|
||||
// First value for `name` (ASCII case-insensitive), or nullopt.
|
||||
[[nodiscard]] std::optional<std::string_view> get(std::string_view name) const {
|
||||
for (const auto &f : fields_)
|
||||
if (iequals(f.name, name))
|
||||
return f.value;
|
||||
return std::nullopt;
|
||||
}
|
||||
[[nodiscard]] bool has(std::string_view name) const { return get(name).has_value(); }
|
||||
|
||||
static bool iequals(std::string_view a, std::string_view b) noexcept {
|
||||
if (a.size() != b.size())
|
||||
return false;
|
||||
for (std::size_t i = 0; i < a.size(); ++i)
|
||||
if (lower(a[i]) != lower(b[i]))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
static constexpr char lower(char c) noexcept {
|
||||
return (c >= 'A' && c <= 'Z') ? char(c - 'A' + 'a') : c;
|
||||
}
|
||||
std::vector<HeaderField> fields_;
|
||||
};
|
||||
|
||||
enum class ProxyKind { none, http, socks5, socks5_hostname };
|
||||
|
||||
struct ProxyConfig {
|
||||
ProxyKind kind = ProxyKind::none;
|
||||
std::string host; // host[:port]
|
||||
std::uint16_t port = 0;
|
||||
std::string username; // empty = no proxy auth
|
||||
std::string password;
|
||||
};
|
||||
|
||||
enum class AuthScheme { none, basic, digest, any };
|
||||
|
||||
struct AuthConfig {
|
||||
AuthScheme scheme = AuthScheme::none;
|
||||
std::string username;
|
||||
std::string password;
|
||||
};
|
||||
|
||||
struct Cookie {
|
||||
std::string name;
|
||||
std::string value;
|
||||
};
|
||||
|
||||
// One HTTP request the engine wants performed. Defaults are the well-behaved case.
|
||||
struct Request {
|
||||
std::string url;
|
||||
Method method = Method::get;
|
||||
std::vector<HeaderField> headers; // verbatim; browser UA/Referer/cookies live here
|
||||
std::optional<ByteRange> range;
|
||||
std::vector<Cookie> cookies; // merged into a Cookie: header + curl's jar
|
||||
std::string user_agent; // convenience; also settable via headers
|
||||
std::string referrer;
|
||||
|
||||
ProxyConfig proxy;
|
||||
AuthConfig auth;
|
||||
|
||||
bool follow_redirects = true;
|
||||
long max_redirects = 20;
|
||||
bool accept_encoding = false; // OFF for downloads: a gzip'd body breaks Range math
|
||||
|
||||
// Stall detection: abort if throughput stays under `low_speed_bytes_per_sec` for
|
||||
// `low_speed_secs`. 0 disables. Distinct from an overall deadline (probe sets one).
|
||||
long connect_timeout_ms = 15000;
|
||||
long low_speed_bytes_per_sec = 1024;
|
||||
long low_speed_secs = 30;
|
||||
long overall_timeout_ms = 0; // 0 = none; probe uses a few seconds
|
||||
|
||||
// Coarse download-rate ceiling handed to curl (CURLOPT_MAX_RECV_SPEED_LARGE). The
|
||||
// precise limiter (stage 7) pauses/resumes on top of this. 0 = unlimited.
|
||||
std::uint64_t max_recv_bytes_per_sec = 0;
|
||||
};
|
||||
|
||||
// Delivered once, when response headers are in.
|
||||
struct ResponseHead {
|
||||
long status = 0;
|
||||
std::string effective_url; // after redirects
|
||||
HeaderList headers;
|
||||
std::optional<std::uint64_t> content_length; // from Content-Length, if present & sane
|
||||
};
|
||||
|
||||
} // namespace vdm::net
|
||||
|
||||
#endif // VDM_NET_HTTP_TYPES_HPP
|
||||
Reference in New Issue
Block a user