sami/vdm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity
Files
eb72aa522ced029b1c0026bf02f90fa707409e08
vdm/gui/src/dialogs/GrabberWizard.cpp
T
samiandClaude Sonnet 5 de83ee3cce gui: Options, Scheduler, Speed Limiter, Batch, Grabber, and the tray icon 
Continues the build order past the Add-URL/File-Info/Progress dialog flow.

- OptionsDialog: General/Save To/Connection/Downloads/Proxy/Sounds tabs, every
  control bound to a real settings.* key (contracts/schema/types/Settings.
  schema.json). The spec's File Types and Site Logins tabs have no settings.*
  backing (categories go through category.upsert, credentials through the
  Secret Service) so they don't exist here — a tab either binds to a real key
  or isn't shipped. diffChanged() sends only what actually changed, matching
  settings.set's "changed[] names exactly what took effect" contract.
- SchedulerDialog: per-queue schedule (schedule.get/set) plus maxConcurrent/
  onComplete (queue.upsert), Start Now/Stop. Queue.schema.json already carries
  the schedule so queue.list alone seeds the window.
- SpeedLimiterDialog: the live global limiter (limiter.get/set) — a different
  thing from Options' downloads.speedLimit* default. buildParams() enforces
  the schema's "0 with enabled true must not be offered".
- BatchDialog: clipboard-blob and {start..end}-wildcard tabs sharing one
  category/queue/start-mode footer into download.addBatch.
- GrabberWizard: 4-step QWizard (project label -> start URL/depth/filters ->
  file-type filter -> review), grabber.start feeding a poll+event.grabber.
  progress-driven review page, Finish = grabber.harvest for the checked files.
- TrayIcon: active-count tooltip, Show/Add URL/Pause All/Resume All/Speed
  Limiter submenu/Quit. Quit only closes the GUI — there is no RPC to stop
  veloxd itself, filed as a new gap in daemon-requests-m1.md. MainWindow now
  also hides to tray instead of closing when general.minimizeToTray is set.

Every dialog's non-widget logic (diffChanged, buildSchedule, buildParams,
parseUrlBlob/expandWildcard/buildAddBatchParams, buildFileTypes/
buildStartParams/buildHarvestParams) is a static pure function with its own
test, same shape as FileInfoDialog::buildSpec from the previous round.

Verified end-to-end against a running mockd under ASan+UBSan: all five
surfaces render real data (settings.get values, queue.list's two seeded
queues, limiter.get, a live grabber.start/status crawl returning 3 files) with
no sanitizer reports. gui-check (non-ASan) and dev (ASan+UBSan) presets both
build the whole repo clean; all gui-labeled ctest targets pass.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01NSCdCWFXBTSBBK3MzWtJiC
2026-09-11 17:17:43 +04:00

444 lines
17 KiB
C++
Raw Blame History

#include "dialogs/GrabberWizard.hpp"
#include <utility>
#include <QAbstractItemView>
#include <QCheckBox>
#include <QComboBox>
#include <QFormLayout>
#include <QHBoxLayout>
#include <QHeaderView>
#include <QJsonArray>
#include <QLabel>
#include <QLineEdit>
#include <QLocale>
#include <QPointer>
#include <QPushButton>
#include <QRegularExpression>
#include <QSpinBox>
#include <QTableWidget>
#include <QTableWidgetItem>
#include <QTimer>
#include <QVBoxLayout>
#include "rpc/Protocol.hpp"
#include "rpc/RpcClient.hpp"
namespace velox::gui {
namespace {
QStringList splitList(const QString &text) {
QStringList out;
for (const QString &raw : text.split(QRegularExpression(QStringLiteral("[,\n]")))) {
const QString s = raw.trimmed();
if (!s.isEmpty()) {
out << s;
}
}
return out;
}
// --- Page 1: project (client-side label only; grabber.start has no template concept) ---
class GrabberProjectPage : public QWizardPage {
public:
explicit GrabberProjectPage(QWidget *parent) : QWizardPage(parent) {
setTitle(QObject::tr("Project"));
setSubTitle(QObject::tr("Give this crawl a name for your own reference."));
name_ = new QLineEdit(this);
name_->setPlaceholderText(QObject::tr("e.g. Photo gallery mirror"));
auto *layout = new QFormLayout(this);
layout->addRow(QObject::tr("Name:"), name_);
}
private:
QLineEdit *name_;
};
// --- Page 2: start URL, depth, host/pattern filters -------------------------------------
class GrabberStartPage : public QWizardPage {
public:
explicit GrabberStartPage(QWidget *parent) : QWizardPage(parent) {
setTitle(QObject::tr("Start URL"));
startUrl_ = new QLineEdit(this);
startUrl_->setPlaceholderText(QStringLiteral("https://example.com/gallery/"));
registerField(QStringLiteral("startUrl*"), startUrl_);
depth_ = new QSpinBox(this);
depth_->setRange(0, 10);
depth_->setValue(1);
sameHostOnly_ = new QCheckBox(QObject::tr("Stay on the same host"), this);
sameHostOnly_->setChecked(true);
includePatterns_ = new QLineEdit(this);
includePatterns_->setPlaceholderText(QObject::tr("comma-separated globs, optional"));
excludePatterns_ = new QLineEdit(this);
excludePatterns_->setPlaceholderText(QObject::tr("comma-separated globs, optional"));
auto *layout = new QFormLayout(this);
layout->addRow(QObject::tr("Start URL:"), startUrl_);
layout->addRow(QObject::tr("Crawl depth:"), depth_);
layout->addRow(sameHostOnly_);
layout->addRow(QObject::tr("Include only:"), includePatterns_);
layout->addRow(QObject::tr("Exclude:"), excludePatterns_);
}
QString startUrl() const { return startUrl_->text().trimmed(); }
int depth() const { return depth_->value(); }
bool sameHostOnly() const { return sameHostOnly_->isChecked(); }
QString includePatterns() const { return includePatterns_->text(); }
QString excludePatterns() const { return excludePatterns_->text(); }
private:
QLineEdit *startUrl_;
QSpinBox *depth_;
QCheckBox *sameHostOnly_;
QLineEdit *includePatterns_;
QLineEdit *excludePatterns_;
};
// --- Page 3: file-type filter ------------------------------------------------------------
class GrabberFileTypesPage : public QWizardPage {
public:
explicit GrabberFileTypesPage(QWidget *parent) : QWizardPage(parent) {
setTitle(QObject::tr("File types"));
setSubTitle(QObject::tr("Leave everything unchecked to crawl every file type."));
images_ = new QCheckBox(QObject::tr("Images (jpg, png, gif, webp)"), this);
archives_ = new QCheckBox(QObject::tr("Archives (zip, rar, 7z, tar, gz)"), this);
docs_ = new QCheckBox(QObject::tr("Documents (pdf, doc, docx, xls, xlsx)"), this);
video_ = new QCheckBox(QObject::tr("Video (mp4, mkv, avi, webm)"), this);
audio_ = new QCheckBox(QObject::tr("Audio (mp3, flac, wav, ogg)"), this);
custom_ = new QLineEdit(this);
custom_->setPlaceholderText(QObject::tr("custom extensions, comma-separated"));
auto *layout = new QVBoxLayout(this);
layout->addWidget(images_);
layout->addWidget(archives_);
layout->addWidget(docs_);
layout->addWidget(video_);
layout->addWidget(audio_);
auto *customRow = new QFormLayout;
customRow->addRow(QObject::tr("Also:"), custom_);
layout->addLayout(customRow);
}
bool images() const { return images_->isChecked(); }
bool archives() const { return archives_->isChecked(); }
bool docs() const { return docs_->isChecked(); }
bool video() const { return video_->isChecked(); }
bool audio() const { return audio_->isChecked(); }
QString customCsv() const { return custom_->text(); }
private:
QCheckBox *images_;
QCheckBox *archives_;
QCheckBox *docs_;
QCheckBox *video_;
QCheckBox *audio_;
QLineEdit *custom_;
};
// --- Page 4: crawl + review + harvest -----------------------------------------------------
class GrabberReviewPage : public QWizardPage {
Q_OBJECT
public:
GrabberReviewPage(rpc::RpcClient *client, QJsonArray categories, GrabberStartPage *startPage,
GrabberFileTypesPage *fileTypesPage, QWidget *parent)
: QWizardPage(parent),
client_(client),
startPage_(startPage),
fileTypesPage_(fileTypesPage) {
setTitle(QObject::tr("Review found files"));
setFinalPage(true);
statusLabel_ = new QLabel(this);
table_ = new QTableWidget(0, 3, this);
table_->setHorizontalHeaderLabels(
{QObject::tr("Get"), QObject::tr("File"), QObject::tr("Size")});
table_->horizontalHeader()->setSectionResizeMode(1, QHeaderView::Stretch);
table_->verticalHeader()->setVisible(false);
table_->setEditTriggers(QAbstractItemView::NoEditTriggers);
auto *selectAll = new QPushButton(QObject::tr("Select All"), this);
auto *selectNone = new QPushButton(QObject::tr("Select None"), this);
connect(selectAll, &QPushButton::clicked, this, [this] { setAllChecked(true); });
connect(selectNone, &QPushButton::clicked, this, [this] { setAllChecked(false); });
auto *selectRow = new QHBoxLayout;
selectRow->addWidget(selectAll);
selectRow->addWidget(selectNone);
selectRow->addStretch();
categoryCombo_ = new QComboBox(this);
categoryCombo_->addItem(QObject::tr("(Automatic)"), QString());
for (const QJsonValue &v : categories) {
const QJsonObject c = v.toObject();
categoryCombo_->addItem(c.value("name").toString(), c.value("categoryId").toString());
}
startModeCombo_ = new QComboBox(this);
startModeCombo_->addItem(QObject::tr("Download Now"), QStringLiteral("now"));
startModeCombo_->addItem(QObject::tr("Download Later"), QStringLiteral("later"));
errorLabel_ = new QLabel(this);
errorLabel_->setStyleSheet(QStringLiteral("color: #c0392b;"));
errorLabel_->hide();
auto *footer = new QFormLayout;
footer->addRow(QObject::tr("Category:"), categoryCombo_);
footer->addRow(QObject::tr("Start mode:"), startModeCombo_);
auto *layout = new QVBoxLayout(this);
layout->addWidget(statusLabel_);
layout->addLayout(selectRow);
layout->addWidget(table_, 1);
layout->addLayout(footer);
layout->addWidget(errorLabel_);
pollTimer_ = new QTimer(this);
pollTimer_->setInterval(5000); // catch-up fallback; live updates come from the event
connect(pollTimer_, &QTimer::timeout, this, &GrabberReviewPage::fetchStatus);
connect(client_, &rpc::RpcClient::grabberProgress, this, &GrabberReviewPage::onProgress);
}
void initializePage() override {
if (started_) {
return;
}
started_ = true;
statusLabel_->setText(QObject::tr("Starting crawl…"));
const QJsonValue fileTypes = GrabberWizard::buildFileTypes(
fileTypesPage_->images(), fileTypesPage_->archives(), fileTypesPage_->docs(),
fileTypesPage_->video(), fileTypesPage_->audio(), fileTypesPage_->customCsv());
QJsonObject params = GrabberWizard::buildStartParams(
startPage_->startUrl(), startPage_->depth(), startPage_->sameHostOnly(),
startPage_->includePatterns(), startPage_->excludePatterns(), fileTypes);
QPointer<GrabberReviewPage> self(this);
client_->call(
QString::fromLatin1(rpc::method::kGrabberStart), params,
[self](const rpc::RpcReply &reply) {
if (!self) {
return;
}
if (!reply.ok()) {
self->statusLabel_->setText(
QObject::tr("Could not start the crawl: %1").arg(reply.error.message));
return;
}
self->jobId_ = reply.result.toObject().value("jobId").toString();
self->pollTimer_->start();
self->fetchStatus();
});
}
bool isComplete() const override { return !files_.isEmpty(); }
/// QWizard calls this on Finish and only advances (closes, for the last page) when it
/// returns true. harvest is async, so this always returns false and closes the wizard
/// itself from the reply callback instead — the same submit-then-accept shape every
/// other dialog here uses, just routed through validatePage() rather than a button.
bool validatePage() override {
if (harvesting_) {
return false;
}
QStringList selected;
for (int row = 0; row < table_->rowCount(); ++row) {
if (table_->item(row, 0)->checkState() == Qt::Checked) {
selected << table_->item(row, 0)->data(Qt::UserRole).toString();
}
}
if (selected.isEmpty()) {
return false;
}
harvesting_ = true;
const QJsonObject params = GrabberWizard::buildHarvestParams(
jobId_, selected, categoryCombo_->currentData().toString(),
startModeCombo_->currentData().toString());
QPointer<GrabberReviewPage> self(this);
client_->call(
QString::fromLatin1(rpc::method::kGrabberHarvest), params,
[self](const rpc::RpcReply &reply) {
if (!self) {
return;
}
self->harvesting_ = false;
if (!reply.ok()) {
self->errorLabel_->setText(QObject::tr("Could not add the selected files: %1")
.arg(reply.error.message));
self->errorLabel_->show();
return;
}
self->wizard()->accept();
});
return false;
}
private:
void setAllChecked(bool checked) {
for (int row = 0; row < table_->rowCount(); ++row) {
table_->item(row, 0)->setCheckState(checked ? Qt::Checked : Qt::Unchecked);
}
}
void onProgress(const QJsonObject &params) {
if (params.value("jobId").toString() != jobId_ || jobId_.isEmpty()) {
return;
}
statusLabel_->setText(QObject::tr("Crawled %1, found %2…")
.arg(params.value("crawled").toInt())
.arg(params.value("found").toInt()));
if (params.value("done").toBool()) {
fetchStatus();
}
}
void fetchStatus() {
if (jobId_.isEmpty()) {
return;
}
QPointer<GrabberReviewPage> self(this);
client_->call(QString::fromLatin1(rpc::method::kGrabberStatus),
QJsonObject{{"jobId", jobId_}}, [self](const rpc::RpcReply &reply) {
if (!self || !reply.ok()) {
return;
}
self->applyStatus(reply.result.toObject());
});
}
void applyStatus(const QJsonObject &status) {
const QString state = status.value("state").toString();
files_ = status.value("files").toArray();
populateTable();
if (state == QLatin1String("crawling")) {
statusLabel_->setText(QObject::tr("Crawling… %1 found so far").arg(files_.size()));
} else if (state == QLatin1String("done")) {
pollTimer_->stop();
statusLabel_->setText(QObject::tr("Done — %1 file(s) found.").arg(files_.size()));
} else {
pollTimer_->stop();
statusLabel_->setText(
QObject::tr("Crawl %1: %2").arg(state, status.value("error").toString()));
}
emit completeChanged();
}
void populateTable() {
table_->setRowCount(files_.size());
for (int row = 0; row < files_.size(); ++row) {
const QJsonObject f = files_[row].toObject();
auto *checkItem =
new QTableWidgetItem(f.value("filename").toString(f.value("url").toString()));
checkItem->setFlags(checkItem->flags() | Qt::ItemIsUserCheckable);
checkItem->setCheckState(Qt::Checked);
checkItem->setData(Qt::UserRole, f.value("fileId").toString());
table_->setItem(row, 0, checkItem);
table_->setItem(row, 1, new QTableWidgetItem(f.value("url").toString()));
const QJsonValue size = f.value("sizeBytes");
table_->setItem(
row, 2,
new QTableWidgetItem(size.isDouble() ? QLocale().formattedDataSize(
static_cast<qint64>(size.toDouble()))
: QObject::tr("unknown")));
}
}
rpc::RpcClient *client_;
GrabberStartPage *startPage_;
GrabberFileTypesPage *fileTypesPage_;
QString jobId_;
QJsonArray files_;
bool started_ = false;
bool harvesting_ = false;
QLabel *statusLabel_;
QTableWidget *table_;
QComboBox *categoryCombo_;
QComboBox *startModeCombo_;
QLabel *errorLabel_;
QTimer *pollTimer_;
};
} // namespace
QJsonValue GrabberWizard::buildFileTypes(bool images, bool archives, bool docs, bool video,
bool audio, const QString &customCsv) {
QJsonArray types;
if (images) {
for (const char *e : {"jpg", "jpeg", "png", "gif", "webp"}) {
types.append(e);
}
}
if (archives) {
for (const char *e : {"zip", "rar", "7z", "tar", "gz"}) {
types.append(e);
}
}
if (docs) {
for (const char *e : {"pdf", "doc", "docx", "xls", "xlsx"}) {
types.append(e);
}
}
if (video) {
for (const char *e : {"mp4", "mkv", "avi", "webm"}) {
types.append(e);
}
}
if (audio) {
for (const char *e : {"mp3", "flac", "wav", "ogg"}) {
types.append(e);
}
}
for (const QString &e : splitList(customCsv)) {
types.append(e);
}
return types.isEmpty() ? QJsonValue(QJsonValue::Null) : QJsonValue(types);
}
QJsonObject GrabberWizard::buildStartParams(const QString &startUrl, int depth, bool sameHostOnly,
const QString &includePatterns,
const QString &excludePatterns,
const QJsonValue &fileTypes) {
const QStringList include = splitList(includePatterns);
const QStringList exclude = splitList(excludePatterns);
return QJsonObject{
{"startUrl", startUrl},
{"depth", depth},
{"sameHostOnly", sameHostOnly},
{"includePatterns", include.isEmpty() ? QJsonValue(QJsonValue::Null)
: QJsonValue(QJsonArray::fromStringList(include))},
{"excludePatterns", exclude.isEmpty() ? QJsonValue(QJsonValue::Null)
: QJsonValue(QJsonArray::fromStringList(exclude))},
{"fileTypes", fileTypes},
};
}
QJsonObject GrabberWizard::buildHarvestParams(const QString &jobId, const QStringList &selectedIds,
const QString &categoryId, const QString &startMode) {
QJsonObject defaults{{"url", QString()}, {"startMode", startMode}}; // url is ignored here
if (!categoryId.isEmpty()) {
defaults["categoryId"] = categoryId;
}
return QJsonObject{
{"jobId", jobId},
{"select", QJsonArray::fromStringList(selectedIds)},
{"defaults", defaults},
};
}
GrabberWizard::GrabberWizard(rpc::RpcClient *client, QJsonArray categories, QWidget *parent)
: QWizard(parent), client_(client), categories_(std::move(categories)) {
setWindowTitle(tr("Site Grabber"));
resize(640, 520);
addPage(new GrabberProjectPage(this));
auto *startPage = new GrabberStartPage(this);
addPage(startPage);
auto *fileTypesPage = new GrabberFileTypesPage(this);
addPage(fileTypesPage);
addPage(new GrabberReviewPage(client_, categories_, startPage, fileTypesPage, this));
}
} // namespace velox::gui
#include "GrabberWizard.moc"
Reference in New Issue View Git Blame Copy Permalink