merge: build system, testserver, CI
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
# Velox C++ style. Owned by lane PKG/QA; CLAUDE.md §6 makes this authoritative.
|
||||
# `clang-format --dry-run -Werror` runs on every PR.
|
||||
---
|
||||
Language: Cpp
|
||||
BasedOnStyle: Google
|
||||
|
||||
IndentWidth: 4
|
||||
ContinuationIndentWidth: 4
|
||||
ColumnLimit: 100
|
||||
AccessModifierOffset: -2
|
||||
NamespaceIndentation: None
|
||||
|
||||
DerivePointerAlignment: false
|
||||
PointerAlignment: Right
|
||||
|
||||
AllowShortFunctionsOnASingleLine: Inline
|
||||
AllowShortIfStatementsOnASingleLine: Never
|
||||
AllowShortLoopsOnASingleLine: false
|
||||
AllowShortCaseLabelsOnASingleLine: false
|
||||
|
||||
IndentCaseLabels: true
|
||||
|
||||
IncludeBlocks: Preserve
|
||||
SortIncludes: CaseSensitive
|
||||
|
||||
# Keep the "} // namespace vdm" trailer style Google uses.
|
||||
FixNamespaceComments: true
|
||||
ShortNamespaceLines: 0
|
||||
---
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
# Velox clang-tidy config. Owned by lane PKG/QA. Runs on every PR (non-fatal advisory in
|
||||
# M0–M1, a required check from M2). Keep the enabled set small and high-signal; add checks
|
||||
# when they catch a real bug, not for completeness.
|
||||
---
|
||||
Checks: >
|
||||
-*,
|
||||
bugprone-*,
|
||||
cert-err34-c,
|
||||
cert-err58-cpp,
|
||||
concurrency-*,
|
||||
cppcoreguidelines-pro-type-member-init,
|
||||
cppcoreguidelines-slicing,
|
||||
cppcoreguidelines-virtual-class-destructor,
|
||||
misc-definitions-in-headers,
|
||||
misc-misplaced-const,
|
||||
misc-unused-using-decls,
|
||||
modernize-use-nullptr,
|
||||
modernize-use-override,
|
||||
modernize-use-using,
|
||||
modernize-use-emplace,
|
||||
modernize-loop-convert,
|
||||
modernize-make-unique,
|
||||
modernize-make-shared,
|
||||
performance-*,
|
||||
portability-*,
|
||||
readability-container-size-empty,
|
||||
readability-duplicate-include,
|
||||
readability-misleading-indentation,
|
||||
readability-redundant-*,
|
||||
readability-simplify-boolean-expr,
|
||||
readability-static-definition-in-anonymous-namespace,
|
||||
-bugprone-easily-swappable-parameters,
|
||||
-bugprone-narrowing-conversions,
|
||||
-performance-avoid-endl,
|
||||
-readability-redundant-access-specifiers
|
||||
|
||||
WarningsAsErrors: ''
|
||||
HeaderFilterRegex: '(core|daemon|cli|nmhost|tools)/.*\.(hpp|h)$'
|
||||
FormatStyle: file
|
||||
|
||||
CheckOptions:
|
||||
- key: performance-move-const-arg.CheckTriviallyCopyableMove
|
||||
value: 'false'
|
||||
- key: bugprone-argument-comment.StrictMode
|
||||
value: 'true'
|
||||
@@ -0,0 +1,39 @@
|
||||
# Velox editor defaults. Owned by lane PKG/QA. Mirrors .clang-format for C++ and the
|
||||
# prevailing conventions for everything else.
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
indent_style = space
|
||||
|
||||
[*.{cpp,cc,hpp,h}]
|
||||
indent_size = 4
|
||||
max_line_length = 100
|
||||
|
||||
[*.{ts,tsx,js,jsx,json,mjs}]
|
||||
indent_size = 2
|
||||
|
||||
[*.{py}]
|
||||
indent_size = 4
|
||||
max_line_length = 100
|
||||
|
||||
[*.{cmake,txt}]
|
||||
indent_size = 4
|
||||
|
||||
[CMakeLists.txt]
|
||||
indent_size = 4
|
||||
|
||||
[*.{yml,yaml}]
|
||||
indent_size = 2
|
||||
|
||||
[*.sh]
|
||||
indent_size = 4
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
@@ -0,0 +1,36 @@
|
||||
# Branch protection for `main`
|
||||
|
||||
CI defines the checks; **branch protection is a repo setting** (Settings → Branches →
|
||||
Add rule) and has to be configured once by an admin. This file records the intended
|
||||
policy so it can be re-applied or audited.
|
||||
|
||||
## Rule: `main`
|
||||
|
||||
- **Require a pull request before merging.** No direct pushes.
|
||||
- **Require status checks to pass before merging**, and require branches to be up to date
|
||||
first. Required checks:
|
||||
|
||||
| Check (job name in `ci.yml`) | Required from |
|
||||
|---|---|
|
||||
| `clang-format` | now |
|
||||
| `testserver` | now |
|
||||
| `bootstrap-script` | now |
|
||||
| `build (gcc)` / `build (clang)` | when the first C++ lane merges |
|
||||
| `sanitizers (dev)` / `sanitizers (tsan)` | when the first C++ lane merges |
|
||||
| `conformance` | **when `tests/conformance/` lands — this is the M0 exit gate** |
|
||||
| `extension-lint` | when `extension/` lands |
|
||||
|
||||
`clang-tidy` is intentionally **not** required through M1 (`continue-on-error: true`,
|
||||
`.clang-tidy` has `WarningsAsErrors: ''`). Make it required at M2.
|
||||
|
||||
- **Require linear history** (matches CLAUDE.md §6: rebase onto `main`, no merge commits).
|
||||
- **Require conversation resolution before merging.**
|
||||
- Do **not** allow force pushes or deletions.
|
||||
- Apply the rule to administrators too, except for the initial scaffolding period.
|
||||
|
||||
## Note on the "skipped" job steps
|
||||
|
||||
Several jobs (`conformance`, `extension-lint`, `clang-tidy`) short-circuit to a "skipped"
|
||||
echo when their lane hasn't landed. They still report **success**, so they can be marked
|
||||
required now without blocking — they start doing real work automatically on the commit
|
||||
that adds the lane.
|
||||
@@ -0,0 +1,166 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# GitHub-hosted runners are Ubuntu 24.04; the project targets 26.04. bootstrap.sh warns
|
||||
# but proceeds. Revisit when 26.04 runners exist.
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
|
||||
jobs:
|
||||
# --- fast lint jobs: no compiler, no heavy deps -------------------------------------
|
||||
clang-format:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install clang-format
|
||||
run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends clang-format
|
||||
- name: Check formatting
|
||||
run: |
|
||||
shopt -s globstar nullglob
|
||||
files=(core/**/*.{cpp,hpp} daemon/**/*.{cpp,hpp} cli/**/*.{cpp,hpp} nmhost/**/*.{cpp,hpp})
|
||||
if [ ${#files[@]} -eq 0 ]; then echo "no C++ sources yet — skipping"; exit 0; fi
|
||||
printf '%s\n' "${files[@]}"
|
||||
clang-format --dry-run --Werror "${files[@]}"
|
||||
|
||||
testserver:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: testserver self-test
|
||||
run: python3 tools/testserver/selftest.py
|
||||
|
||||
bootstrap-script:
|
||||
# Keeps tools/bootstrap.sh honest: it must run clean and its --check must pass.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: sudo ./tools/bootstrap.sh --with-clang
|
||||
- run: ./tools/bootstrap.sh --check
|
||||
|
||||
extension-lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- id: check
|
||||
run: |
|
||||
if [ -f extension/package.json ]; then echo "present=true" >> "$GITHUB_OUTPUT"
|
||||
else echo "present=false" >> "$GITHUB_OUTPUT"; fi
|
||||
- uses: actions/setup-node@v4
|
||||
if: steps.check.outputs.present == 'true'
|
||||
with:
|
||||
node-version: '20'
|
||||
- name: web-ext lint
|
||||
if: steps.check.outputs.present == 'true'
|
||||
working-directory: extension
|
||||
run: |
|
||||
npm ci
|
||||
npx web-ext lint --source-dir .
|
||||
- name: skipped
|
||||
if: steps.check.outputs.present == 'false'
|
||||
run: echo "extension/ has not landed yet — skipping web-ext lint"
|
||||
|
||||
# --- build + test matrix ----------------------------------------------------------
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
compiler: [gcc, clang]
|
||||
env:
|
||||
CC: ${{ matrix.compiler == 'gcc' && 'gcc' || 'clang' }}
|
||||
CXX: ${{ matrix.compiler == 'gcc' && 'g++' || 'clang++' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Bootstrap toolchain
|
||||
run: sudo ./tools/bootstrap.sh --with-clang
|
||||
- name: Configure
|
||||
run: cmake --preset ci
|
||||
- name: Build
|
||||
run: cmake --build --preset ci
|
||||
- name: Test
|
||||
run: ctest --preset ci --output-on-failure
|
||||
|
||||
sanitizers:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
preset: [dev, tsan] # dev = ASan + UBSan
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Bootstrap toolchain
|
||||
run: sudo ./tools/bootstrap.sh --with-clang
|
||||
- name: Configure
|
||||
run: cmake --preset ${{ matrix.preset }}
|
||||
- name: Build
|
||||
run: cmake --build --preset ${{ matrix.preset }}
|
||||
- name: Test
|
||||
run: ctest --preset ${{ matrix.preset }} --output-on-failure
|
||||
env:
|
||||
ASAN_OPTIONS: detect_leaks=1:halt_on_error=1
|
||||
UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
|
||||
TSAN_OPTIONS: halt_on_error=1
|
||||
|
||||
clang-tidy:
|
||||
# Advisory through M1 (see .clang-tidy WarningsAsErrors: ''); becomes required at M2.
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- id: check
|
||||
run: |
|
||||
if ls core/CMakeLists.txt daemon/CMakeLists.txt >/dev/null 2>&1; then
|
||||
echo "present=true" >> "$GITHUB_OUTPUT"
|
||||
else echo "present=false" >> "$GITHUB_OUTPUT"; fi
|
||||
- name: Bootstrap toolchain
|
||||
if: steps.check.outputs.present == 'true'
|
||||
run: sudo ./tools/bootstrap.sh
|
||||
- name: Configure (for compile_commands.json)
|
||||
if: steps.check.outputs.present == 'true'
|
||||
run: cmake --preset dev
|
||||
- name: Run clang-tidy on changed files
|
||||
if: steps.check.outputs.present == 'true'
|
||||
run: |
|
||||
mapfile -t files < <(git diff --name-only --diff-filter=ACM \
|
||||
"${{ github.event.pull_request.base.sha || 'HEAD~1' }}" HEAD \
|
||||
-- '*.cpp' '*.hpp' || true)
|
||||
[ ${#files[@]} -eq 0 ] && { echo "no C++ changes"; exit 0; }
|
||||
printf '%s\n' "${files[@]}"
|
||||
clang-tidy -p build/dev "${files[@]}"
|
||||
- name: skipped
|
||||
if: steps.check.outputs.present == 'false'
|
||||
run: echo "no C++ lane has landed a CMakeLists yet — skipping clang-tidy"
|
||||
|
||||
conformance:
|
||||
# Required check on every PR once tests/conformance/ lands (branch protection is
|
||||
# configured in the repo settings, not here — see .github/BRANCH_PROTECTION.md).
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- id: check
|
||||
run: |
|
||||
if [ -f tests/conformance/CMakeLists.txt ] || [ -f tests/conformance/package.json ]; then
|
||||
echo "present=true" >> "$GITHUB_OUTPUT"
|
||||
else echo "present=false" >> "$GITHUB_OUTPUT"; fi
|
||||
- name: Bootstrap toolchain
|
||||
if: steps.check.outputs.present == 'true'
|
||||
run: sudo ./tools/bootstrap.sh
|
||||
- name: Run conformance suite
|
||||
if: steps.check.outputs.present == 'true'
|
||||
run: |
|
||||
cmake --preset dev
|
||||
ctest --preset dev --output-on-failure -L conformance
|
||||
- name: skipped
|
||||
if: steps.check.outputs.present == 'false'
|
||||
run: echo "tests/conformance/ has not landed yet — skipping"
|
||||
+28
-4
@@ -1,15 +1,39 @@
|
||||
# Build output
|
||||
build/
|
||||
.cache/
|
||||
build-*/
|
||||
/out/
|
||||
compile_commands.json
|
||||
CMakeUserPresets.json
|
||||
.cache/
|
||||
.ccls-cache/
|
||||
|
||||
# Language / tooling
|
||||
node_modules/
|
||||
dist/
|
||||
web-ext-artifacts/
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Compiled objects
|
||||
*.o
|
||||
*.so
|
||||
*.a
|
||||
*.log
|
||||
.venv/
|
||||
__pycache__/
|
||||
|
||||
# Editor / OS
|
||||
.idea/
|
||||
.vscode/
|
||||
*.user
|
||||
*.orig
|
||||
*.swp
|
||||
.DS_Store
|
||||
|
||||
# Profiling
|
||||
perf.data*
|
||||
callgrind.out.*
|
||||
massif.out.*
|
||||
|
||||
# Runtime / logs / partial downloads
|
||||
*.log
|
||||
*.veloxpart
|
||||
*.veloxpart.meta
|
||||
|
||||
+88
-29
@@ -1,6 +1,11 @@
|
||||
# Velox Download Manager — top-level build
|
||||
# SCAFFOLDING ONLY. Lane PKG/QA owns this file; each add_subdirectory() is enabled by the
|
||||
# owning lane when it lands its first target. Nothing here builds yet by design.
|
||||
# Velox Download Manager — top-level build. Owned by lane PKG/QA.
|
||||
#
|
||||
# Lanes land in parallel and merge to main independently, so every add_subdirectory()
|
||||
# here is guarded by EXISTS on the lane's own CMakeLists.txt: main always configures,
|
||||
# and a lane's targets light up the moment that lane merges — no coordinated edit here.
|
||||
#
|
||||
# cmake --preset dev && cmake --build --preset dev
|
||||
# ctest --preset dev
|
||||
|
||||
cmake_minimum_required(VERSION 3.28)
|
||||
|
||||
@@ -14,35 +19,89 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
option(VELOX_BUILD_GUI "Build the Qt 6 GUI client" ON)
|
||||
option(VELOX_BUILD_TESTS "Build unit and integration tests" ON)
|
||||
option(VELOX_BUILD_FUZZ "Build libFuzzer targets (clang only)" OFF)
|
||||
option(VELOX_ENABLE_MEDIA "Build the HLS/DASH media grabber (M4)" OFF)
|
||||
if(NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY)
|
||||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
|
||||
endif()
|
||||
|
||||
option(VELOX_BUILD_GUI "Build the Qt 6 GUI client" ON)
|
||||
option(VELOX_BUILD_TESTS "Build unit and integration tests" ON)
|
||||
option(VELOX_BUILD_FUZZ "Build libFuzzer targets (clang only)" OFF)
|
||||
option(VELOX_ENABLE_MEDIA "Build the HLS/DASH media grabber (M4)" OFF)
|
||||
option(VELOX_WERROR "Treat warnings as errors" ON)
|
||||
|
||||
# Project-wide warning flags. Set via add_compile_options (a directory property), not the
|
||||
# CMAKE_CXX_FLAGS cache variable, because the dev/tsan presets overwrite that cache var
|
||||
# wholesale for sanitizer flags — a target's warnings must not ride on it.
|
||||
add_compile_options(-Wall -Wextra -Wpedantic)
|
||||
if(CMAKE_BUILD_TYPE STREQUAL "Release" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
|
||||
if(VELOX_WERROR)
|
||||
add_compile_options(-Werror)
|
||||
endif()
|
||||
|
||||
# --- Dependencies (see README bootstrap; none are vendored) --------------------
|
||||
# find_package(CURL 8.0 REQUIRED)
|
||||
# find_package(SQLite3 REQUIRED)
|
||||
# find_package(nlohmann_json 3.11 REQUIRED)
|
||||
# find_package(OpenSSL REQUIRED)
|
||||
# if(VELOX_BUILD_GUI)
|
||||
# find_package(Qt6 6.6 REQUIRED COMPONENTS Widgets Svg Network LinguistTools)
|
||||
# qt_standard_project_setup()
|
||||
# endif()
|
||||
if(VELOX_BUILD_FUZZ AND NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")
|
||||
message(WARNING "VELOX_BUILD_FUZZ is ON but the compiler is ${CMAKE_CXX_COMPILER_ID}; "
|
||||
"libFuzzer needs Clang. Fuzz targets will be skipped.")
|
||||
endif()
|
||||
|
||||
# --- Targets: each lane enables its own line ----------------------------------
|
||||
# add_subdirectory(core) # lane CORE → libveloxcore
|
||||
# add_subdirectory(daemon) # lane DAEMON → veloxd
|
||||
# add_subdirectory(cli) # lane DAEMON → velox
|
||||
# add_subdirectory(nmhost) # lane DAEMON → velox-nmhost
|
||||
# if(VELOX_BUILD_GUI)
|
||||
# add_subdirectory(gui) # lane GUI → velox-gui
|
||||
# endif()
|
||||
# if(VELOX_BUILD_TESTS)
|
||||
# enable_testing()
|
||||
# add_subdirectory(tests)
|
||||
# endif()
|
||||
# --- Dependencies -------------------------------------------------------------------
|
||||
# Found once here so a missing -dev package fails at configure with a clear name, rather
|
||||
# than deep in a lane. Each find is gated on the lane that needs it being present.
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/core/CMakeLists.txt)
|
||||
find_package(CURL 8.0 REQUIRED)
|
||||
find_package(OpenSSL REQUIRED)
|
||||
endif()
|
||||
|
||||
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/daemon/CMakeLists.txt)
|
||||
find_package(SQLite3 REQUIRED)
|
||||
find_package(nlohmann_json 3.11 REQUIRED)
|
||||
find_package(PkgConfig REQUIRED)
|
||||
pkg_check_modules(LIBSECRET REQUIRED IMPORTED_TARGET libsecret-1)
|
||||
endif()
|
||||
|
||||
if(VELOX_BUILD_GUI AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/gui/CMakeLists.txt)
|
||||
find_package(Qt6 6.6 REQUIRED COMPONENTS Widgets Svg Network LinguistTools)
|
||||
qt_standard_project_setup()
|
||||
endif()
|
||||
|
||||
if(VELOX_ENABLE_MEDIA)
|
||||
find_package(PkgConfig REQUIRED)
|
||||
pkg_check_modules(FFMPEG REQUIRED IMPORTED_TARGET
|
||||
libavformat libavcodec libavutil)
|
||||
endif()
|
||||
|
||||
# --- Tests --------------------------------------------------------------------------
|
||||
if(VELOX_BUILD_TESTS)
|
||||
enable_testing()
|
||||
endif()
|
||||
|
||||
# --- Lane targets (guarded; see header) -------------------------------------------
|
||||
foreach(lane IN ITEMS core daemon cli nmhost)
|
||||
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${lane}/CMakeLists.txt)
|
||||
add_subdirectory(${lane})
|
||||
else()
|
||||
message(STATUS "velox: lane '${lane}' has not landed a CMakeLists yet — skipping.")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
if(VELOX_BUILD_GUI)
|
||||
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/gui/CMakeLists.txt)
|
||||
add_subdirectory(gui)
|
||||
else()
|
||||
message(STATUS "velox: lane 'gui' has not landed a CMakeLists yet — skipping.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
foreach(toolset tools/testserver tools/bench tools/fuzz tools/mockd)
|
||||
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${toolset}/CMakeLists.txt)
|
||||
add_subdirectory(${toolset})
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
if(VELOX_BUILD_TESTS)
|
||||
foreach(suite tests/conformance tests/integration tests/e2e)
|
||||
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${suite}/CMakeLists.txt)
|
||||
add_subdirectory(${suite})
|
||||
endif()
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
@@ -58,6 +58,18 @@
|
||||
"configurePreset": "dev",
|
||||
"output": { "outputOnFailure": true },
|
||||
"execution": { "noTestsAction": "error", "stopOnFailure": false }
|
||||
},
|
||||
{
|
||||
"name": "tsan",
|
||||
"configurePreset": "tsan",
|
||||
"output": { "outputOnFailure": true },
|
||||
"execution": { "noTestsAction": "error", "stopOnFailure": false }
|
||||
},
|
||||
{
|
||||
"name": "ci",
|
||||
"configurePreset": "ci",
|
||||
"output": { "outputOnFailure": true },
|
||||
"execution": { "noTestsAction": "error", "stopOnFailure": false }
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Executable
+208
@@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# tools/bootstrap.sh — install every build/test dependency for Velox Download Manager.
|
||||
#
|
||||
# Target: a clean Ubuntu 26.04 LTS machine. Safe to re-run (apt is idempotent).
|
||||
# Owned by lane PKG/QA. Keep this in sync with the table in README.md "Toolchain bootstrap".
|
||||
#
|
||||
# ./tools/bootstrap.sh # everything needed to build + test
|
||||
# ./tools/bootstrap.sh --with-clang # also install clang/LLVM for the libFuzzer targets
|
||||
# ./tools/bootstrap.sh --packaging # also install .deb tooling (lane PKG, M6)
|
||||
# ./tools/bootstrap.sh --check # verify versions only, install nothing
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
WITH_CLANG=0
|
||||
WITH_PACKAGING=0
|
||||
CHECK_ONLY=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--with-clang) WITH_CLANG=1 ;;
|
||||
--packaging) WITH_PACKAGING=1 ;;
|
||||
--check) CHECK_ONLY=1 ;;
|
||||
-h|--help)
|
||||
sed -n '3,12p' "$0" | sed 's/^# \{0,1\}//'
|
||||
exit 0 ;;
|
||||
*) echo "bootstrap: unknown option '$arg'" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
log() { printf '\033[1;34m==>\033[0m %s\n' "$*"; }
|
||||
warn() { printf '\033[1;33mwarning:\033[0m %s\n' "$*" >&2; }
|
||||
die() { printf '\033[1;31merror:\033[0m %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
# --- sanity --------------------------------------------------------------------
|
||||
|
||||
[[ "$(uname -s)" == "Linux" ]] || die "this project targets Linux (Ubuntu 26.04)."
|
||||
|
||||
if [[ -r /etc/os-release ]]; then
|
||||
. /etc/os-release
|
||||
if [[ "${ID:-}" != "ubuntu" ]]; then
|
||||
warn "distro is '${ID:-unknown}', not ubuntu — package names may differ."
|
||||
elif [[ "${VERSION_ID:-}" != "26.04" ]]; then
|
||||
warn "Ubuntu ${VERSION_ID:-?} detected; the project is developed against 26.04."
|
||||
fi
|
||||
fi
|
||||
|
||||
command -v apt-get >/dev/null 2>&1 || die "apt-get not found; install deps manually (see README)."
|
||||
|
||||
SUDO=""
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
command -v sudo >/dev/null 2>&1 || die "not root and no sudo; re-run as root."
|
||||
SUDO="sudo"
|
||||
fi
|
||||
|
||||
# --- package sets -----------------------------------------------------------------
|
||||
# Full list, so a clean machine gets everything even though the dev box already has most.
|
||||
|
||||
APT_CORE=(
|
||||
build-essential # g++, make
|
||||
g++ # C++23 front end (need >= 13; 26.04 ships 15)
|
||||
gdb
|
||||
cmake # >= 3.28 (26.04 ships 4.2)
|
||||
ninja-build
|
||||
pkg-config
|
||||
git
|
||||
python3 # testserver, conformance helpers, codegen
|
||||
ca-certificates
|
||||
curl
|
||||
)
|
||||
|
||||
APT_LIBS=(
|
||||
libcurl4-openssl-dev # CORE: HTTP stack
|
||||
libsqlite3-dev # DAEMON: task store
|
||||
nlohmann-json3-dev # DAEMON/PROTO: JSON-RPC (never in core/)
|
||||
libssl-dev # CORE: SHA-256 / TLS bits
|
||||
libsecret-1-dev # DAEMON: Secret Service for credentials
|
||||
libavformat-dev # MEDIA (M4): HLS/DASH mux
|
||||
libavcodec-dev
|
||||
libavutil-dev
|
||||
ffmpeg # MEDIA (M4): runtime muxer
|
||||
)
|
||||
|
||||
APT_GUI=(
|
||||
qt6-base-dev
|
||||
qt6-tools-dev
|
||||
qt6-tools-dev-tools # lrelease/lupdate for i18n
|
||||
libqt6svg6-dev # GUI: SVG icon rendering
|
||||
)
|
||||
|
||||
APT_LINT=(
|
||||
clang-format
|
||||
clang-tidy
|
||||
)
|
||||
|
||||
APT_NODE=(
|
||||
nodejs # EXT + tools/mockd + conformance runner
|
||||
npm
|
||||
)
|
||||
|
||||
APT_CLANG=(
|
||||
clang # optional: libFuzzer targets (M7)
|
||||
llvm
|
||||
)
|
||||
|
||||
APT_PACKAGING=(
|
||||
debhelper
|
||||
dpkg-dev
|
||||
lintian
|
||||
devscripts
|
||||
fakeroot
|
||||
)
|
||||
|
||||
# --- install --------------------------------------------------------------------
|
||||
|
||||
PKGS=("${APT_CORE[@]}" "${APT_LIBS[@]}" "${APT_GUI[@]}" "${APT_LINT[@]}" "${APT_NODE[@]}")
|
||||
[[ $WITH_CLANG -eq 1 ]] && PKGS+=("${APT_CLANG[@]}")
|
||||
[[ $WITH_PACKAGING -eq 1 ]] && PKGS+=("${APT_PACKAGING[@]}")
|
||||
|
||||
if [[ $CHECK_ONLY -eq 0 ]]; then
|
||||
log "apt-get update"
|
||||
$SUDO apt-get update -qq
|
||||
log "installing ${#PKGS[@]} packages (idempotent; already-present ones are skipped)"
|
||||
DEBIAN_FRONTEND=noninteractive $SUDO apt-get install -y --no-install-recommends "${PKGS[@]}"
|
||||
else
|
||||
log "--check: skipping installation"
|
||||
fi
|
||||
|
||||
# --- verify --------------------------------------------------------------------
|
||||
|
||||
log "verifying toolchain"
|
||||
|
||||
fail=0
|
||||
need_cmd() {
|
||||
if command -v "$1" >/dev/null 2>&1; then
|
||||
printf ' \033[32m✓\033[0m %-16s %s\n' "$1" "$("${@:2}" 2>&1 | head -1)"
|
||||
else
|
||||
printf ' \033[31m✗\033[0m %-16s MISSING\n' "$1"
|
||||
fail=1
|
||||
fi
|
||||
}
|
||||
|
||||
need_cmd git git --version
|
||||
need_cmd cmake cmake --version
|
||||
need_cmd ninja ninja --version
|
||||
need_cmd g++ g++ --version
|
||||
need_cmd pkg-config pkg-config --version
|
||||
need_cmd python3 python3 --version
|
||||
need_cmd clang-format clang-format --version
|
||||
need_cmd clang-tidy clang-tidy --version
|
||||
need_cmd node node --version
|
||||
need_cmd npm npm --version
|
||||
[[ $WITH_CLANG -eq 1 ]] && need_cmd clang clang --version
|
||||
|
||||
# Version floors that actually matter.
|
||||
if command -v cmake >/dev/null 2>&1; then
|
||||
cmake_ver="$(cmake --version | sed -n '1s/.* //p')"
|
||||
# `sort -V -C` exits 0 iff the input is already in ascending order, i.e. floor <= ver.
|
||||
if ! printf '3.28.0\n%s\n' "$cmake_ver" | sort -V -C; then
|
||||
warn "cmake $cmake_ver is below the 3.28 floor in CMakeLists.txt."
|
||||
fi
|
||||
# CMake >= 4 hard-errors on cmake_minimum_required() < 3.5 in any dependency.
|
||||
if printf '4.0.0\n%s\n' "$cmake_ver" | sort -V -C; then
|
||||
log "cmake $cmake_ver is 4.x — dependencies with a pre-3.5 CMake floor will fail to configure."
|
||||
fi
|
||||
fi
|
||||
|
||||
if command -v g++ >/dev/null 2>&1; then
|
||||
gxx_major="$(g++ -dumpversion | cut -d. -f1)"
|
||||
if (( gxx_major < 13 )); then
|
||||
warn "g++ $gxx_major lacks the C++23 support this project needs (>= 13)."
|
||||
fi
|
||||
echo '#include <expected>
|
||||
#include <version>
|
||||
#if __cpp_lib_expected < 202202L
|
||||
#error no std::expected
|
||||
#endif
|
||||
int main(){}' > /tmp/vdm_bootstrap_cxx23.$$.cpp
|
||||
if g++ -std=c++23 -fsyntax-only /tmp/vdm_bootstrap_cxx23.$$.cpp 2>/dev/null; then
|
||||
printf ' \033[32m✓\033[0m %-16s std::expected / -std=c++23 OK\n' "c++23"
|
||||
else
|
||||
printf ' \033[31m✗\033[0m %-16s -std=c++23 with <expected> failed\n' "c++23"
|
||||
fail=1
|
||||
fi
|
||||
rm -f /tmp/vdm_bootstrap_cxx23.$$.cpp
|
||||
fi
|
||||
|
||||
if command -v node >/dev/null 2>&1; then
|
||||
node_major="$(node --version | sed 's/^v//; s/\..*//')"
|
||||
if (( node_major < 20 )); then
|
||||
warn "node $node_major may be too old for the extension toolchain; consider nvm (>= 20)."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Qt / library dev packages: check via pkg-config where possible.
|
||||
for mod in Qt6Core Qt6Widgets Qt6Svg libcurl sqlite3 libssl libsecret-1 libavformat; do
|
||||
if pkg-config --exists "$mod" 2>/dev/null; then
|
||||
printf ' \033[32m✓\033[0m %-16s %s\n' "$mod" "$(pkg-config --modversion "$mod")"
|
||||
else
|
||||
printf ' \033[31m✗\033[0m %-16s pkg-config can'\''t find it\n' "$mod"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
if [[ $fail -ne 0 ]]; then
|
||||
die "toolchain incomplete — see the ✗ lines above."
|
||||
fi
|
||||
log "toolchain OK. Next: cmake --preset dev && cmake --build --preset dev"
|
||||
@@ -0,0 +1,21 @@
|
||||
# tools/testserver — no build, just register the self-test with CTest so a broken
|
||||
# hostile-mode contract is caught in CI, not in a CORE debugging session.
|
||||
|
||||
if(NOT VELOX_BUILD_TESTS)
|
||||
return()
|
||||
endif()
|
||||
|
||||
find_package(Python3 3.11 COMPONENTS Interpreter)
|
||||
if(NOT Python3_Interpreter_FOUND)
|
||||
message(WARNING "testserver: Python 3.11+ not found; skipping its self-test.")
|
||||
return()
|
||||
endif()
|
||||
|
||||
add_test(
|
||||
NAME testserver_selftest
|
||||
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/selftest.py
|
||||
)
|
||||
set_tests_properties(testserver_selftest PROPERTIES
|
||||
LABELS "tools;qa"
|
||||
TIMEOUT 120
|
||||
)
|
||||
@@ -0,0 +1,63 @@
|
||||
# tools/testserver — the hostile HTTP server
|
||||
|
||||
Every failure mode a real download hits, reproducible on demand. Standard-library Python
|
||||
only (>= 3.11); no `pip install`. Owned by lane PKG/QA. **CORE's M1 definition of done is
|
||||
written in terms of this server.**
|
||||
|
||||
```bash
|
||||
tools/testserver/testserver.py --port 8080
|
||||
tools/testserver/testserver.py --port 0 # ephemeral; the chosen port is printed to stdout
|
||||
python3 tools/testserver/selftest.py # smoke-test every mode (also a CTest)
|
||||
```
|
||||
|
||||
## Request shape
|
||||
|
||||
```
|
||||
GET /<mode>[+<mode>...]/file/<size>
|
||||
GET /<mode>/sha256/<size> -> {"sha256": "<hex>", "size": <n>} reference digest
|
||||
GET /<mode>/sign/<size>?ttl=<sec> -> {"url": "...", "exp": <unixtime>} (signed-URL mode)
|
||||
```
|
||||
|
||||
`<size>` is a byte count with an optional `K`/`M`/`G` suffix (binary: KiB/MiB/GiB) —
|
||||
`1048576`, `64K`, `512M`, `5G`. The body is **deterministic**: byte *i* is a function of
|
||||
`(--seed, path, i)`, so any range is independently verifiable and the whole file has a
|
||||
stable SHA-256 (fetch it from the `/sha256/` route). Different paths/modes have different
|
||||
content; a given path is byte-stable across requests except where a mode says otherwise
|
||||
(`etag-changes` still serves stable bytes; only its ETag moves).
|
||||
|
||||
Combine modes with `+`: `/throttled+etag-changes/file/1G`.
|
||||
|
||||
## Modes
|
||||
|
||||
| Mode | Behaviour |
|
||||
|---|---|
|
||||
| *(omitted)* / `plain` | Well-behaved: honours `Range`, stable `ETag`, correct `Content-Length`, `Accept-Ranges: bytes`. |
|
||||
| `no-range` | No `Accept-Ranges`; `Range` ignored; always `200` full body. |
|
||||
| `lies-about-accept-ranges` | Advertises `Accept-Ranges: bytes` but ignores `Range` and returns `200`. |
|
||||
| `etag-changes` | `ETag` differs on every response. A `Range` with a non-matching `If-Range` gets `200` full — i.e. "the file changed under you". |
|
||||
| `flaky-reset` | Sends ~half the requested bytes then aborts the connection with a TCP **RST**. The 1st and 2nd attempt for a given `path+range` fail this way; the 3rd succeeds cleanly. `POST /__control {"reset":true}` clears the counters. |
|
||||
| `slow-loris` | Status line, headers, and the first bytes are dribbled out one byte at a time for `--loris-seconds`, then the rest streams normally. `Connection: close`. |
|
||||
| `redirect-chain` | `302` `--redirect-depth` times (default 5) before the real resource. Query string is preserved across hops. |
|
||||
| `401-basic` | HTTP Basic; credentials `test` / `test`. |
|
||||
| `401-digest` | HTTP Digest, `qop=auth`; credentials `test` / `test`. |
|
||||
| `403-without-referer` | `403` unless `Referer` names this server's origin; otherwise serves normally. |
|
||||
| `416-always` | Any `Range` request → `416` with `Content-Range: bytes */<size>`. A plain `GET` still returns `200` so the re-probe path is exercised. |
|
||||
| `content-length-mismatch` | `Content-Length` header is correct-looking but the server sends ~half and closes unclean. |
|
||||
| `expiring-signed-url` | Requires `?exp=&sig=`. Past `exp` → `403 {"error":"expired"}`; bad/missing sig → `403`. Get a fresh URL from `/…/sign/<size>?ttl=<sec>`. Pair with `download.refreshUrl`. |
|
||||
| `throttled` | Body rate-limited to `--throttle-bps` (default 1 MiB/s). |
|
||||
| `chunked-no-length` | `Transfer-Encoding: chunked`, no `Content-Length` — size unknown until the stream ends. |
|
||||
| `utf8-content-disposition` | `Content-Disposition: attachment; filename="rates.pdf"; filename*=UTF-8''%E2%82%AC%20rates.pdf` (RFC 5987 → "€ rates.pdf"). |
|
||||
| `legacy-content-disposition` | `Content-Disposition` with an RFC 2047 MIME encoded-word filename — the classic mojibake source. |
|
||||
|
||||
## Control & health
|
||||
|
||||
| Route | |
|
||||
|---|---|
|
||||
| `GET /__health` | `200 ok` |
|
||||
| `POST /__control` `{"reset": true}` | Clears per-path attempt counters (`flaky-reset`). Call it between test cases. |
|
||||
|
||||
## Flags
|
||||
|
||||
`--host` (default `127.0.0.1`) · `--port` (`0` = ephemeral) · `--seed` (content seed,
|
||||
default `1`) · `--loris-seconds` (default `5`) · `--redirect-depth` (default `5`) ·
|
||||
`--throttle-bps` (default `1048576`) · `--verbose`.
|
||||
Executable
+291
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Smoke test for testserver.py — every hostile mode answers the way its contract says.
|
||||
|
||||
Not a substitute for CORE's own conformance against it; this just guards the server from
|
||||
regressing. Run directly (`tools/testserver/selftest.py`) or via ctest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
SERVER = HERE / "testserver.py"
|
||||
SEED = 42
|
||||
|
||||
_fail = 0
|
||||
|
||||
|
||||
def check(name: str, cond: bool, detail: str = "") -> None:
|
||||
global _fail
|
||||
mark = "\033[32mok\033[0m" if cond else "\033[31mFAIL\033[0m"
|
||||
print(f" {mark} {name}" + (f" — {detail}" if detail and not cond else ""))
|
||||
if not cond:
|
||||
_fail += 1
|
||||
|
||||
|
||||
class Server:
|
||||
def __init__(self) -> None:
|
||||
self.proc = subprocess.Popen(
|
||||
[sys.executable, str(SERVER), "--port", "0", "--seed", str(SEED),
|
||||
"--loris-seconds", "1", "--throttle-bps", str(256 * 1024)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True,
|
||||
)
|
||||
self.port = int(self.proc.stdout.readline().strip())
|
||||
for _ in range(50):
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", self.port), timeout=0.2):
|
||||
break
|
||||
except OSError:
|
||||
time.sleep(0.05)
|
||||
|
||||
def conn(self) -> http.client.HTTPConnection:
|
||||
return http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||||
|
||||
def get(self, path: str, headers: dict | None = None):
|
||||
c = self.conn()
|
||||
c.request("GET", path, headers=headers or {})
|
||||
r = c.getresponse()
|
||||
body = r.read()
|
||||
c.close()
|
||||
return r, body
|
||||
|
||||
def raw_get(self, path: str, headers: dict | None = None) -> tuple[int, dict, bytes, bool]:
|
||||
"""Returns (status, headers, body_bytes, clean_eof). Tolerates a mid-body RST."""
|
||||
s = socket.create_connection(("127.0.0.1", self.port), timeout=10)
|
||||
req = f"GET {path} HTTP/1.1\r\nHost: 127.0.0.1:{self.port}\r\nConnection: close\r\n"
|
||||
for k, v in (headers or {}).items():
|
||||
req += f"{k}: {v}\r\n"
|
||||
req += "\r\n"
|
||||
s.sendall(req.encode())
|
||||
chunks = []
|
||||
clean = True
|
||||
try:
|
||||
while True:
|
||||
b = s.recv(65536)
|
||||
if not b:
|
||||
break
|
||||
chunks.append(b)
|
||||
except ConnectionResetError:
|
||||
clean = False
|
||||
finally:
|
||||
s.close()
|
||||
raw = b"".join(chunks)
|
||||
head, _, body = raw.partition(b"\r\n\r\n")
|
||||
lines = head.split(b"\r\n")
|
||||
status = int(lines[0].split()[1]) if lines and lines[0] else 0
|
||||
hdrs = {}
|
||||
for ln in lines[1:]:
|
||||
if b":" in ln:
|
||||
k, v = ln.split(b":", 1)
|
||||
hdrs[k.decode().strip().lower()] = v.decode().strip()
|
||||
return status, hdrs, body, clean
|
||||
|
||||
def close(self) -> None:
|
||||
self.proc.terminate()
|
||||
try:
|
||||
self.proc.wait(timeout=3)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.proc.kill()
|
||||
|
||||
|
||||
def ref_sha(srv: Server, path_mode: str, size_spec: str) -> tuple[str, int]:
|
||||
r, body = srv.get(f"/{path_mode}/sha256/{size_spec}")
|
||||
j = json.loads(body)
|
||||
return j["sha256"], j["size"]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
srv = Server()
|
||||
print(f"testserver on :{srv.port} seed={SEED}")
|
||||
try:
|
||||
# health
|
||||
r, body = srv.get("/__health")
|
||||
check("health", r.status == 200 and body == b"ok")
|
||||
|
||||
# plain: full body matches the reference hash, Range works, 206 + Content-Range
|
||||
want, size = ref_sha(srv, "plain", "256K")
|
||||
r, body = srv.get("/plain/file/256K")
|
||||
check("plain full 200", r.status == 200 and len(body) == size)
|
||||
check("plain sha256 matches", hashlib.sha256(body).hexdigest() == want)
|
||||
check("plain advertises Accept-Ranges", r.getheader("Accept-Ranges") == "bytes")
|
||||
r, body = srv.get("/plain/file/256K", {"Range": "bytes=1000-1999"})
|
||||
check("plain range 206", r.status == 206 and len(body) == 1000)
|
||||
check("plain Content-Range", r.getheader("Content-Range") == f"bytes 1000-1999/{size}")
|
||||
check("plain range bytes correct", body == full_slice(srv, "plain", "256K", 1000, 2000))
|
||||
|
||||
# no-range: no Accept-Ranges, Range ignored -> 200 full
|
||||
r, body = srv.get("/no-range/file/128K", {"Range": "bytes=0-1023"})
|
||||
check("no-range ignores Range (200)", r.status == 200 and len(body) == 128 * 1024)
|
||||
check("no-range hides Accept-Ranges", r.getheader("Accept-Ranges") is None)
|
||||
|
||||
# lies-about-accept-ranges: advertises, ignores
|
||||
r, body = srv.get("/lies-about-accept-ranges/file/128K", {"Range": "bytes=0-1023"})
|
||||
check("lies advertises Accept-Ranges", r.getheader("Accept-Ranges") == "bytes")
|
||||
check("lies still sends 200 full", r.status == 200 and len(body) == 128 * 1024)
|
||||
|
||||
# etag-changes: ETag differs across requests; If-Range mismatch -> 200
|
||||
r1, _ = srv.get("/etag-changes/file/64K")
|
||||
r2, _ = srv.get("/etag-changes/file/64K")
|
||||
check("etag-changes: ETag varies", r1.getheader("ETag") != r2.getheader("ETag"))
|
||||
r, body = srv.get("/etag-changes/file/64K",
|
||||
{"Range": "bytes=0-99", "If-Range": '"stale"'})
|
||||
check("etag-changes: If-Range mismatch -> 200 full", r.status == 200 and len(body) == 64 * 1024)
|
||||
|
||||
# 416-always: Range -> 416 with Content-Range */size; plain GET -> 200
|
||||
r, body = srv.get("/416-always/file/64K", {"Range": "bytes=0-99"})
|
||||
check("416-always range -> 416", r.status == 416)
|
||||
check("416-always Content-Range */n", r.getheader("Content-Range") == f"bytes */{64*1024}")
|
||||
r, body = srv.get("/416-always/file/64K")
|
||||
check("416-always plain GET -> 200", r.status == 200 and len(body) == 64 * 1024)
|
||||
|
||||
# redirect-chain: 302s then 200 (http.client doesn't auto-follow; do it by hand)
|
||||
status, body, hops = follow(srv, "/redirect-chain/file/16K")
|
||||
check("redirect-chain lands 200", status == 200 and len(body) == 16 * 1024)
|
||||
check("redirect-chain actually bounced", hops >= 3, f"{hops} hops")
|
||||
|
||||
# 401-basic
|
||||
r, _ = srv.get("/401-basic/file/16K")
|
||||
check("401-basic challenges", r.status == 401 and "Basic" in (r.getheader("WWW-Authenticate") or ""))
|
||||
import base64 as _b64
|
||||
tok = _b64.b64encode(b"test:test").decode()
|
||||
r, body = srv.get("/401-basic/file/16K", {"Authorization": f"Basic {tok}"})
|
||||
check("401-basic accepts test:test", r.status == 200 and len(body) == 16 * 1024)
|
||||
|
||||
# 403-without-referer
|
||||
r, _ = srv.get("/403-without-referer/file/16K")
|
||||
check("403 without Referer", r.status == 403)
|
||||
r, body = srv.get("/403-without-referer/file/16K",
|
||||
{"Referer": f"http://127.0.0.1:{srv.port}/page"})
|
||||
check("403 mode ok with Referer", r.status == 200 and len(body) == 16 * 1024)
|
||||
|
||||
# content-length-mismatch: header length > bytes delivered, unclean close
|
||||
st, hdrs, body, clean = srv.raw_get("/content-length-mismatch/file/64K")
|
||||
check("clen-mismatch: declared > received",
|
||||
int(hdrs.get("content-length", "0")) > len(body), f"{hdrs.get('content-length')} vs {len(body)}")
|
||||
check("clen-mismatch: connection not clean", clean is False)
|
||||
|
||||
# flaky-reset: first two attempts cut + RST, third is clean & correct
|
||||
want, size = ref_sha(srv, "flaky-reset", "64K")
|
||||
results = [srv.raw_get("/flaky-reset/file/64K") for _ in range(3)]
|
||||
check("flaky-reset: attempt 1 unclean", results[0][3] is False and len(results[0][2]) < size)
|
||||
check("flaky-reset: attempt 2 unclean", results[1][3] is False)
|
||||
check("flaky-reset: attempt 3 clean & full", results[2][3] is True and len(results[2][2]) == size)
|
||||
check("flaky-reset: attempt 3 hash ok", hashlib.sha256(results[2][2]).hexdigest() == want)
|
||||
|
||||
# chunked-no-length
|
||||
st, hdrs, body, clean = srv.raw_get("/chunked-no-length/file/32K")
|
||||
check("chunked: TE chunked, no CL",
|
||||
hdrs.get("transfer-encoding") == "chunked" and "content-length" not in hdrs)
|
||||
dechunked = dechunk(body)
|
||||
want, size = ref_sha(srv, "chunked-no-length", "32K")
|
||||
check("chunked: body decodes to full file", len(dechunked) == size)
|
||||
check("chunked: hash ok", hashlib.sha256(dechunked).hexdigest() == want)
|
||||
|
||||
# throttled: 256K at 256 KiB/s budget -> takes >= ~0.9s
|
||||
t0 = time.monotonic()
|
||||
r, body = srv.get("/throttled/file/512K")
|
||||
dt = time.monotonic() - t0
|
||||
check("throttled: rate-limited", dt >= 0.9 and len(body) == 512 * 1024, f"{dt:.2f}s")
|
||||
|
||||
# slow-loris: eventually completes, slowly
|
||||
t0 = time.monotonic()
|
||||
st, hdrs, body, clean = srv.raw_get("/slow-loris/file/8K")
|
||||
dt = time.monotonic() - t0
|
||||
check("slow-loris: completes", len(body) == 8 * 1024, f"got {len(body)}")
|
||||
check("slow-loris: was slow", dt >= 0.5, f"{dt:.2f}s")
|
||||
|
||||
# utf8 / legacy content-disposition
|
||||
r, _ = srv.get("/utf8-content-disposition/file/8K")
|
||||
cd = r.getheader("Content-Disposition") or ""
|
||||
check("utf8 CD has filename*=UTF-8''", "filename*=UTF-8''%E2%82%AC" in cd)
|
||||
r, _ = srv.get("/legacy-content-disposition/file/8K")
|
||||
cd = r.getheader("Content-Disposition") or ""
|
||||
check("legacy CD has encoded-word", "=?UTF-8?B?" in cd)
|
||||
|
||||
# expiring-signed-url
|
||||
r, body = srv.get("/expiring-signed-url/sign/8K?ttl=2")
|
||||
signed = json.loads(body)["url"]
|
||||
path = signed.split(f":{srv.port}", 1)[1]
|
||||
r, body = srv.get(path)
|
||||
check("signed URL works before expiry", r.status == 200 and len(body) == 8 * 1024)
|
||||
r, _ = srv.get("/expiring-signed-url/file/8K")
|
||||
check("unsigned request rejected", r.status == 403)
|
||||
time.sleep(2.1)
|
||||
r, body = srv.get(path)
|
||||
check("signed URL rejected after expiry", r.status == 403 and b"expired" in body)
|
||||
|
||||
# __control reset: prime the counter to attempt 2, reset, next attempt is 1 again
|
||||
# (unclean) rather than 3 (which would be clean).
|
||||
srv.raw_get("/flaky-reset/file/4K") # attempt 1
|
||||
srv.raw_get("/flaky-reset/file/4K") # attempt 2
|
||||
_, _, _, clean_before = srv.raw_get("/flaky-reset/file/4K") # attempt 3 -> clean
|
||||
check("flaky-reset: 3rd attempt would be clean", clean_before is True)
|
||||
srv.raw_get("/flaky-reset/file/4K") # attempt 1 again after this reset...
|
||||
c = srv.conn()
|
||||
payload = b'{"reset":true}'
|
||||
c.request("POST", "/__control", body=payload,
|
||||
headers={"Content-Length": str(len(payload))})
|
||||
rr = c.getresponse()
|
||||
rr.read()
|
||||
c.close()
|
||||
check("__control reset 200", rr.status == 200)
|
||||
_, _, _, clean1 = srv.raw_get("/flaky-reset/file/4K") # attempt 1 post-reset
|
||||
_, _, _, clean2 = srv.raw_get("/flaky-reset/file/4K") # attempt 2 post-reset
|
||||
check("__control reset the counter (1st post-reset unclean)", clean1 is False)
|
||||
check("__control reset the counter (2nd post-reset unclean)", clean2 is False)
|
||||
|
||||
finally:
|
||||
srv.close()
|
||||
|
||||
print()
|
||||
if _fail:
|
||||
print(f"\033[31m{_fail} check(s) failed\033[0m")
|
||||
return 1
|
||||
print("\033[32mall checks passed\033[0m")
|
||||
return 0
|
||||
|
||||
|
||||
def full_slice(srv: Server, mode: str, size_spec: str, lo: int, hi: int) -> bytes:
|
||||
r, body = srv.get(f"/{mode}/file/{size_spec}", {"Range": f"bytes={lo}-{hi-1}"})
|
||||
return body
|
||||
|
||||
|
||||
def follow(srv: Server, path: str, limit: int = 10) -> tuple[int, bytes, int]:
|
||||
"""Manually follow 3xx Location headers. Returns (final_status, body, hop_count)."""
|
||||
hops = 0
|
||||
for _ in range(limit):
|
||||
r, body = srv.get(path)
|
||||
if r.status in (301, 302, 303, 307, 308):
|
||||
loc = r.getheader("Location") or ""
|
||||
path = loc.split(f":{srv.port}", 1)[1] if f":{srv.port}" in loc else loc
|
||||
hops += 1
|
||||
continue
|
||||
return r.status, body, hops
|
||||
return 0, b"", hops
|
||||
|
||||
|
||||
def dechunk(raw: bytes) -> bytes:
|
||||
out = bytearray()
|
||||
i = 0
|
||||
while i < len(raw):
|
||||
j = raw.find(b"\r\n", i)
|
||||
if j < 0:
|
||||
break
|
||||
n = int(raw[i:j].split(b";")[0], 16)
|
||||
if n == 0:
|
||||
break
|
||||
out += raw[j + 2:j + 2 + n]
|
||||
i = j + 2 + n + 2
|
||||
return bytes(out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+610
@@ -0,0 +1,610 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Velox hostile HTTP test server.
|
||||
|
||||
Every failure mode a real-world download hits, on purpose, so CORE's resume / retry /
|
||||
segmentation logic can be tested without hunting for a broken CDN. Standard library only
|
||||
(Python >= 3.11); no pip install, ever.
|
||||
|
||||
tools/testserver/testserver.py --port 8080
|
||||
tools/testserver/testserver.py --port 0 # ephemeral; prints the port to stdout
|
||||
|
||||
Request shape
|
||||
-------------
|
||||
GET /<mode>[+<mode>...]/file/<size>
|
||||
|
||||
`<size>` is a byte count with an optional K/M/G suffix (KiB/MiB/GiB): 1048576, 64K, 512M,
|
||||
5G. The body is deterministic — byte i is `prng(seed, path, i)` — so a client can request
|
||||
any range and verify it, and check the whole file against:
|
||||
|
||||
GET /<mode>/sha256/<size> -> {"sha256": "<hex>", "size": <n>}
|
||||
|
||||
`<mode>` (omit, or use `plain`, for the well-behaved server):
|
||||
|
||||
no-range no Accept-Ranges, Range ignored, always 200 full body
|
||||
lies-about-accept-ranges advertises Accept-Ranges: bytes but ignores Range (200)
|
||||
etag-changes ETag differs every response; If-Range mismatch -> 200 full
|
||||
flaky-reset TCP RST partway through the body; succeeds on the 3rd try
|
||||
slow-loris headers/body dribbled out for --loris-seconds, then normal
|
||||
redirect-chain 302 x --redirect-depth before the real resource
|
||||
401-basic Basic auth, credentials test:test
|
||||
401-digest Digest auth (qop=auth), credentials test:test
|
||||
403-without-referer 403 unless Referer names this server's origin
|
||||
416-always Range -> 416; plain GET -> 200 (exercises the re-probe path)
|
||||
content-length-mismatch Content-Length lies; connection closes short
|
||||
expiring-signed-url needs ?exp=&sig=; past exp -> 403. See /<mode>/sign/<size>
|
||||
throttled body rate-limited to --throttle-bps
|
||||
chunked-no-length Transfer-Encoding: chunked, no Content-Length
|
||||
utf8-content-disposition Content-Disposition filename*=UTF-8''... (RFC 5987)
|
||||
legacy-content-disposition MIME encoded-word + raw latin-1 filename (mojibake bait)
|
||||
|
||||
Control
|
||||
-------
|
||||
GET /__health -> 200 "ok"
|
||||
POST /__control {"reset": true} clears per-path attempt counters (flaky-reset,
|
||||
redirect-chain retry state) between test cases
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
# --- deterministic synthetic content -------------------------------------------------
|
||||
|
||||
CHUNK = 64 * 1024
|
||||
|
||||
|
||||
def _size_to_bytes(spec: str) -> int:
|
||||
spec = spec.strip()
|
||||
mult = 1
|
||||
if spec and spec[-1] in "kKmMgG":
|
||||
mult = {"k": 1024, "m": 1024**2, "g": 1024**3}[spec[-1].lower()]
|
||||
spec = spec[:-1]
|
||||
n = int(spec)
|
||||
if n < 0:
|
||||
raise ValueError("negative size")
|
||||
return n * mult
|
||||
|
||||
|
||||
def _stream_body(seed: int, tag: str, start: int, end: int):
|
||||
"""Yield the deterministic bytes for [start, end). Keyed by (seed, tag) so different
|
||||
paths have different content but a given path is stable across requests and ranges."""
|
||||
key = hashlib.sha256(f"{seed}:{tag}".encode()).digest()
|
||||
pos = start
|
||||
block = start // CHUNK
|
||||
while pos < end:
|
||||
h = hashlib.sha256(key + struct.pack("<Q", block)).digest()
|
||||
buf = (h * (CHUNK // len(h) + 1))[:CHUNK]
|
||||
off = pos - block * CHUNK
|
||||
take = min(CHUNK - off, end - pos)
|
||||
yield buf[off:off + take]
|
||||
pos += take
|
||||
block += 1
|
||||
|
||||
|
||||
def _full_sha256(seed: int, tag: str, size: int) -> str:
|
||||
d = hashlib.sha256()
|
||||
for part in _stream_body(seed, tag, 0, size):
|
||||
d.update(part)
|
||||
return d.hexdigest()
|
||||
|
||||
|
||||
# --- per-path attempt state (for flaky modes) --------------------------------------
|
||||
|
||||
class Attempts:
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._n: dict[str, int] = {}
|
||||
|
||||
def bump(self, key: str) -> int:
|
||||
with self._lock:
|
||||
self._n[key] = self._n.get(key, 0) + 1
|
||||
return self._n[key]
|
||||
|
||||
def reset(self) -> None:
|
||||
with self._lock:
|
||||
self._n.clear()
|
||||
|
||||
|
||||
# --- request handler --------------------------------------------------------------
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "veloxtestserver/1.0"
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
# injected by make_server()
|
||||
seed: int = 0
|
||||
loris_seconds: float = 5.0
|
||||
redirect_depth: int = 5
|
||||
throttle_bps: int = 1024 * 1024
|
||||
attempts: Attempts = Attempts()
|
||||
verbose: bool = False
|
||||
|
||||
# -- logging -------------------------------------------------------------------
|
||||
def log_message(self, fmt: str, *args) -> None:
|
||||
if self.verbose:
|
||||
sys.stderr.write(" %s - %s\n" % (self.address_string(), fmt % args))
|
||||
|
||||
# -- helpers ------------------------------------------------------------------
|
||||
def _parts(self):
|
||||
u = urlsplit(self.path)
|
||||
segs = [s for s in u.path.split("/") if s != ""]
|
||||
query = parse_qs(u.query)
|
||||
return segs, query
|
||||
|
||||
def _origin(self) -> str:
|
||||
host = self.headers.get("Host", f"127.0.0.1:{self.server.server_address[1]}")
|
||||
return f"http://{host}"
|
||||
|
||||
def _send_simple(self, status: int, body: bytes = b"", ctype: str = "text/plain"):
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
|
||||
def _reset_connection(self) -> None:
|
||||
"""Abort with a TCP RST rather than a clean FIN, the way a flaky CDN drops you."""
|
||||
try:
|
||||
self.connection.setsockopt(
|
||||
socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)
|
||||
)
|
||||
self.connection.close()
|
||||
except OSError:
|
||||
pass
|
||||
self.close_connection = True
|
||||
|
||||
# -- entry points -----------------------------------------------------------
|
||||
def do_HEAD(self) -> None:
|
||||
self._dispatch()
|
||||
|
||||
def do_GET(self) -> None:
|
||||
self._dispatch()
|
||||
|
||||
def do_POST(self) -> None:
|
||||
segs, _ = self._parts()
|
||||
if segs == ["__control"]:
|
||||
length = int(self.headers.get("Content-Length", "0") or "0")
|
||||
raw = self.rfile.read(length) if length else b"{}"
|
||||
try:
|
||||
msg = json.loads(raw or b"{}")
|
||||
except json.JSONDecodeError:
|
||||
self._send_simple(400, b"bad json")
|
||||
return
|
||||
if msg.get("reset"):
|
||||
self.attempts.reset()
|
||||
self._send_simple(200, b'{"ok":true}', "application/json")
|
||||
return
|
||||
self._send_simple(405, b"method not allowed")
|
||||
|
||||
# -- routing --------------------------------------------------------------
|
||||
def _dispatch(self) -> None:
|
||||
try:
|
||||
segs, query = self._parts()
|
||||
except ValueError:
|
||||
self._send_simple(400, b"bad path")
|
||||
return
|
||||
|
||||
if segs == ["__health"]:
|
||||
self._send_simple(200, b"ok")
|
||||
return
|
||||
if not segs:
|
||||
self._send_simple(200, b"velox testserver: see --help / module docstring")
|
||||
return
|
||||
|
||||
# /redirect/<n>/<rest...> — internal hop target for redirect-chain
|
||||
if segs[0] == "redirect":
|
||||
self._serve_redirect_hop(segs, query)
|
||||
return
|
||||
|
||||
modes = set(segs[0].split("+")) if segs[0] not in ("file", "sha256", "sign") else {"plain"}
|
||||
rest = segs if segs[0] in ("file", "sha256", "sign") else segs[1:]
|
||||
if not rest:
|
||||
self._send_simple(400, b"expected /<mode>/file/<size>")
|
||||
return
|
||||
|
||||
kind = rest[0]
|
||||
arg = rest[1] if len(rest) > 1 else ""
|
||||
|
||||
if kind == "sha256":
|
||||
try:
|
||||
size = _size_to_bytes(arg)
|
||||
except ValueError:
|
||||
self._send_simple(400, b"bad size")
|
||||
return
|
||||
tag = f"{'+'.join(sorted(modes))}/file/{arg}"
|
||||
body = json.dumps({"sha256": _full_sha256(self.seed, tag, size), "size": size}).encode()
|
||||
self._send_simple(200, body, "application/json")
|
||||
return
|
||||
|
||||
if kind == "sign":
|
||||
self._serve_sign(modes, arg, query)
|
||||
return
|
||||
|
||||
if kind != "file":
|
||||
self._send_simple(404, b"unknown resource")
|
||||
return
|
||||
|
||||
try:
|
||||
size = _size_to_bytes(arg)
|
||||
except ValueError:
|
||||
self._send_simple(400, b"bad size")
|
||||
return
|
||||
|
||||
self._serve_file(modes, arg, size, query)
|
||||
|
||||
# -- redirect-chain ------------------------------------------------------
|
||||
def _serve_redirect_hop(self, segs: list[str], query) -> None:
|
||||
# /redirect/<n>/<mode>/file/<size>
|
||||
try:
|
||||
n = int(segs[1])
|
||||
except (IndexError, ValueError):
|
||||
self._send_simple(400, b"bad redirect hop")
|
||||
return
|
||||
tail = "/".join(segs[2:])
|
||||
q = urlsplit(self.path).query
|
||||
qs = f"?{q}" if q else ""
|
||||
self.send_response(302)
|
||||
if n <= 0:
|
||||
self.send_header("Location", f"{self._origin()}/{tail}{qs}")
|
||||
else:
|
||||
self.send_header("Location", f"{self._origin()}/redirect/{n - 1}/{tail}{qs}")
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
# -- signed URL helper -------------------------------------------------
|
||||
def _sign(self, path: str, exp: int) -> str:
|
||||
return hmac.new(
|
||||
struct.pack("<q", self.seed), f"{path}:{exp}".encode(), hashlib.sha256
|
||||
).hexdigest()
|
||||
|
||||
def _serve_sign(self, modes: set[str], arg: str, query) -> None:
|
||||
ttl = int((query.get("ttl", ["10"])[0]))
|
||||
exp = int(time.time()) + ttl
|
||||
path = f"/{'+'.join(sorted(modes))}/file/{arg}"
|
||||
sig = self._sign(path, exp)
|
||||
url = f"{self._origin()}{path}?exp={exp}&sig={sig}"
|
||||
self._send_simple(200, json.dumps({"url": url, "exp": exp}).encode(), "application/json")
|
||||
|
||||
# -- the main file route ---------------------------------------------
|
||||
def _serve_file(self, modes: set[str], size_spec: str, size: int, query) -> None:
|
||||
tag = f"{'+'.join(sorted(modes))}/file/{size_spec}"
|
||||
path_key = urlsplit(self.path).path
|
||||
|
||||
# --- auth gates (checked before anything else) ---
|
||||
if "401-basic" in modes and not self._basic_ok():
|
||||
self.send_response(401)
|
||||
self.send_header("WWW-Authenticate", 'Basic realm="velox-test"')
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
if "401-digest" in modes and not self._digest_ok():
|
||||
nonce = hashlib.md5(f"{time.time()}:{self.seed}".encode()).hexdigest()
|
||||
self.send_response(401)
|
||||
self.send_header(
|
||||
"WWW-Authenticate",
|
||||
f'Digest realm="velox-test", qop="auth", nonce="{nonce}", '
|
||||
f'opaque="{hashlib.md5(b"velox").hexdigest()}"',
|
||||
)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
if "403-without-referer" in modes:
|
||||
ref = self.headers.get("Referer", "")
|
||||
if not ref or self._origin() not in ref:
|
||||
self._send_simple(403, b"referer required")
|
||||
return
|
||||
if "expiring-signed-url" in modes:
|
||||
exp = query.get("exp", [None])[0]
|
||||
sig = query.get("sig", [None])[0]
|
||||
if exp is None or sig is None:
|
||||
self._send_simple(403, b'{"error":"unsigned"}', "application/json")
|
||||
return
|
||||
want = self._sign(f"/{'+'.join(sorted(modes))}/file/{size_spec}", int(exp))
|
||||
if not hmac.compare_digest(sig, want):
|
||||
self._send_simple(403, b'{"error":"bad signature"}', "application/json")
|
||||
return
|
||||
if int(exp) < time.time():
|
||||
self._send_simple(403, b'{"error":"expired"}', "application/json")
|
||||
return
|
||||
|
||||
# --- redirect-chain: bounce before serving ---
|
||||
if "redirect-chain" in modes and query.get("_r", ["0"])[0] != "done":
|
||||
depth = self.redirect_depth
|
||||
tail = f"{'+'.join(sorted(modes))}/file/{size_spec}"
|
||||
self.send_response(302)
|
||||
self.send_header("Location", f"{self._origin()}/redirect/{depth - 1}/{tail}?_r=done")
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
# --- 416-always ---
|
||||
rng = self._parse_range(size)
|
||||
if "416-always" in modes and rng is not None:
|
||||
self.send_response(416)
|
||||
self.send_header("Content-Range", f"bytes */{size}")
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
# --- ranges honoured? ---
|
||||
honour_range = not ({"no-range", "lies-about-accept-ranges"} & modes)
|
||||
advertise_ar = "no-range" not in modes
|
||||
|
||||
etag = self._etag(tag, changing="etag-changes" in modes)
|
||||
if rng is not None and honour_range and "etag-changes" in modes:
|
||||
if_range = self.headers.get("If-Range")
|
||||
if if_range and if_range != etag:
|
||||
rng = None # validator failed -> full 200, mirrors a changed file
|
||||
|
||||
start, end = (0, size)
|
||||
partial = False
|
||||
if rng is not None and honour_range:
|
||||
start, end = rng
|
||||
partial = True
|
||||
|
||||
status = 206 if partial else 200
|
||||
body_len = end - start
|
||||
|
||||
# --- content-length-mismatch: declare a wrong length ---
|
||||
declared_len = body_len
|
||||
short_by = 0
|
||||
if "content-length-mismatch" in modes:
|
||||
short_by = min(4096, body_len // 2 + 1)
|
||||
declared_len = body_len # header says the real length...
|
||||
body_len_to_send = body_len - short_by # ...but we send less and hang up
|
||||
|
||||
# --- flaky-reset: fail the first two attempts ---
|
||||
if "flaky-reset" in modes:
|
||||
n = self.attempts.bump(f"reset:{path_key}:{start}-{end}")
|
||||
if n % 3 != 0:
|
||||
cut = max(1, (end - start) // 2)
|
||||
self.send_response(status)
|
||||
self._common_headers(size, start, end, partial, advertise_ar, etag,
|
||||
declared_len)
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self._write_body(tag, start, start + cut, throttle=False)
|
||||
self._reset_connection()
|
||||
return
|
||||
|
||||
# --- chunked-no-length ---
|
||||
if "chunked-no-length" in modes and not partial:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Transfer-Encoding", "chunked")
|
||||
self._content_disposition(modes)
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
for part in self._body_iter(tag, start, end,
|
||||
throttle="throttled" in modes):
|
||||
self.wfile.write(b"%X\r\n%s\r\n" % (len(part), part))
|
||||
self.wfile.write(b"0\r\n\r\n")
|
||||
return
|
||||
|
||||
# --- slow-loris: dribble the response ---
|
||||
if "slow-loris" in modes:
|
||||
self._serve_loris(tag, size, start, end, partial, advertise_ar, etag)
|
||||
return
|
||||
|
||||
# --- normal (well-behaved, or throttled, or lying-about-length) path ---
|
||||
self.send_response(status)
|
||||
self._common_headers(size, start, end, partial, advertise_ar, etag, declared_len)
|
||||
self._content_disposition(modes)
|
||||
self.end_headers()
|
||||
if self.command == "HEAD":
|
||||
return
|
||||
if "content-length-mismatch" in modes:
|
||||
self._write_body(tag, start, start + body_len_to_send, throttle=False)
|
||||
self._reset_connection()
|
||||
return
|
||||
self._write_body(tag, start, end, throttle="throttled" in modes)
|
||||
|
||||
# -- header / body plumbing ----------------------------------------
|
||||
def _common_headers(self, size, start, end, partial, advertise_ar, etag, declared_len):
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
if advertise_ar:
|
||||
self.send_header("Accept-Ranges", "bytes")
|
||||
self.send_header("ETag", etag)
|
||||
self.send_header("Last-Modified", "Wed, 01 Jan 2025 00:00:00 GMT")
|
||||
if partial:
|
||||
self.send_header("Content-Range", f"bytes {start}-{end - 1}/{size}")
|
||||
self.send_header("Content-Length", str(declared_len))
|
||||
|
||||
def _content_disposition(self, modes: set[str]) -> None:
|
||||
if "utf8-content-disposition" in modes:
|
||||
# € rates.pdf
|
||||
self.send_header(
|
||||
"Content-Disposition",
|
||||
"attachment; filename=\"rates.pdf\"; "
|
||||
"filename*=UTF-8''%E2%82%AC%20rates.pdf",
|
||||
)
|
||||
elif "legacy-content-disposition" in modes:
|
||||
# MIME encoded-word (RFC 2047) + a raw Latin-1 fallback: classic mojibake bait
|
||||
self.send_header(
|
||||
"Content-Disposition",
|
||||
'attachment; filename="=?UTF-8?B?xI1lc2vDoS1zbcOsxJlz.pdf?="',
|
||||
)
|
||||
|
||||
def _body_iter(self, tag: str, start: int, end: int, throttle: bool):
|
||||
budget = self.throttle_bps
|
||||
window_start = time.monotonic()
|
||||
sent_in_window = 0
|
||||
for part in _stream_body(self.seed, tag, start, end):
|
||||
if throttle:
|
||||
sent_in_window += len(part)
|
||||
if sent_in_window >= budget:
|
||||
elapsed = time.monotonic() - window_start
|
||||
if elapsed < 1.0:
|
||||
time.sleep(1.0 - elapsed)
|
||||
window_start = time.monotonic()
|
||||
sent_in_window = 0
|
||||
yield part
|
||||
|
||||
def _write_body(self, tag: str, start: int, end: int, throttle: bool) -> None:
|
||||
try:
|
||||
for part in self._body_iter(tag, start, end, throttle):
|
||||
self.wfile.write(part)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
self.close_connection = True
|
||||
|
||||
def _serve_loris(self, tag, size, start, end, partial, advertise_ar, etag) -> None:
|
||||
deadline = time.monotonic() + self.loris_seconds
|
||||
status_line = f"HTTP/1.1 {206 if partial else 200} X\r\n"
|
||||
self.wfile.write(status_line.encode())
|
||||
headers = [
|
||||
("Content-Type", "application/octet-stream"),
|
||||
("ETag", etag),
|
||||
]
|
||||
if advertise_ar:
|
||||
headers.append(("Accept-Ranges", "bytes"))
|
||||
if partial:
|
||||
headers.append(("Content-Range", f"bytes {start}-{end - 1}/{size}"))
|
||||
headers.append(("Content-Length", str(end - start)))
|
||||
headers.append(("Connection", "close"))
|
||||
self.close_connection = True
|
||||
for k, v in headers:
|
||||
line = f"{k}: {v}\r\n".encode()
|
||||
for b in line:
|
||||
self.wfile.write(bytes([b]))
|
||||
self.wfile.flush()
|
||||
if time.monotonic() < deadline:
|
||||
time.sleep(0.2)
|
||||
self.wfile.write(b"\r\n")
|
||||
if self.command == "HEAD":
|
||||
return
|
||||
slow = True
|
||||
for part in _stream_body(self.seed, tag, start, end):
|
||||
if slow and time.monotonic() < deadline:
|
||||
for b in part:
|
||||
self.wfile.write(bytes([b]))
|
||||
self.wfile.flush()
|
||||
time.sleep(0.05)
|
||||
if time.monotonic() >= deadline:
|
||||
slow = False
|
||||
break
|
||||
else:
|
||||
try:
|
||||
self.wfile.write(part)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
return
|
||||
|
||||
# -- auth --------------------------------------------------------------
|
||||
def _basic_ok(self) -> bool:
|
||||
h = self.headers.get("Authorization", "")
|
||||
if not h.startswith("Basic "):
|
||||
return False
|
||||
try:
|
||||
user, _, pw = base64.b64decode(h[6:]).decode().partition(":")
|
||||
except Exception:
|
||||
return False
|
||||
return user == "test" and pw == "test"
|
||||
|
||||
def _digest_ok(self) -> bool:
|
||||
h = self.headers.get("Authorization", "")
|
||||
if not h.startswith("Digest "):
|
||||
return False
|
||||
params = {}
|
||||
for item in h[7:].split(","):
|
||||
if "=" not in item:
|
||||
continue
|
||||
k, v = item.strip().split("=", 1)
|
||||
params[k] = v.strip('"')
|
||||
need = {"username", "realm", "nonce", "uri", "response"}
|
||||
if not need.issubset(params) or params["username"] != "test":
|
||||
return False
|
||||
ha1 = hashlib.md5(f"test:{params['realm']}:test".encode()).hexdigest()
|
||||
ha2 = hashlib.md5(f"{self.command}:{params['uri']}".encode()).hexdigest()
|
||||
if params.get("qop") == "auth":
|
||||
resp = hashlib.md5(
|
||||
f"{ha1}:{params['nonce']}:{params.get('nc','')}:"
|
||||
f"{params.get('cnonce','')}:auth:{ha2}".encode()
|
||||
).hexdigest()
|
||||
else:
|
||||
resp = hashlib.md5(f"{ha1}:{params['nonce']}:{ha2}".encode()).hexdigest()
|
||||
return hmac.compare_digest(resp, params["response"])
|
||||
|
||||
# -- misc ------------------------------------------------------------
|
||||
def _etag(self, tag: str, changing: bool) -> str:
|
||||
if changing:
|
||||
return '"' + hashlib.md5(f"{tag}:{time.time_ns()}".encode()).hexdigest() + '"'
|
||||
return '"' + hashlib.md5(tag.encode()).hexdigest() + '"'
|
||||
|
||||
def _parse_range(self, size: int):
|
||||
h = self.headers.get("Range")
|
||||
if not h or not h.startswith("bytes="):
|
||||
return None
|
||||
spec = h[6:].split(",")[0].strip()
|
||||
try:
|
||||
if spec.startswith("-"):
|
||||
n = int(spec[1:])
|
||||
return (max(0, size - n), size)
|
||||
lo_s, _, hi_s = spec.partition("-")
|
||||
lo = int(lo_s)
|
||||
hi = int(hi_s) + 1 if hi_s else size
|
||||
if lo >= size or lo < 0 or hi > size or lo >= hi:
|
||||
return None
|
||||
return (lo, hi)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def make_server(host: str, port: int, args) -> ThreadingHTTPServer:
|
||||
attempts = Attempts()
|
||||
|
||||
class Bound(Handler):
|
||||
pass
|
||||
|
||||
Bound.seed = args.seed
|
||||
Bound.loris_seconds = args.loris_seconds
|
||||
Bound.redirect_depth = args.redirect_depth
|
||||
Bound.throttle_bps = args.throttle_bps
|
||||
Bound.attempts = attempts
|
||||
Bound.verbose = args.verbose
|
||||
|
||||
httpd = ThreadingHTTPServer((host, port), Bound)
|
||||
httpd.daemon_threads = True
|
||||
return httpd
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(description="Velox hostile HTTP test server")
|
||||
p.add_argument("--host", default="127.0.0.1")
|
||||
p.add_argument("--port", type=int, default=8080, help="0 for an ephemeral port")
|
||||
p.add_argument("--seed", type=int, default=1, help="deterministic content seed")
|
||||
p.add_argument("--loris-seconds", type=float, default=5.0)
|
||||
p.add_argument("--redirect-depth", type=int, default=5)
|
||||
p.add_argument("--throttle-bps", type=int, default=1024 * 1024)
|
||||
p.add_argument("--verbose", action="store_true")
|
||||
args = p.parse_args()
|
||||
|
||||
httpd = make_server(args.host, args.port, args)
|
||||
actual_port = httpd.server_address[1]
|
||||
print(f"{actual_port}", flush=True)
|
||||
sys.stderr.write(
|
||||
f"velox testserver on http://{args.host}:{actual_port} seed={args.seed}\n"
|
||||
)
|
||||
try:
|
||||
httpd.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user