#include "dialogs/GrabberWizard.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "rpc/Protocol.hpp" #include "rpc/RpcClient.hpp" namespace velox::gui { namespace { QStringList splitList(const QString &text) { QStringList out; for (const QString &raw : text.split(QRegularExpression(QStringLiteral("[,\n]")))) { const QString s = raw.trimmed(); if (!s.isEmpty()) { out << s; } } return out; } // --- Page 1: project (client-side label only; grabber.start has no template concept) --- class GrabberProjectPage : public QWizardPage { public: explicit GrabberProjectPage(QWidget *parent) : QWizardPage(parent) { setTitle(QObject::tr("Project")); setSubTitle(QObject::tr("Give this crawl a name for your own reference.")); name_ = new QLineEdit(this); name_->setPlaceholderText(QObject::tr("e.g. Photo gallery mirror")); auto *layout = new QFormLayout(this); layout->addRow(QObject::tr("Name:"), name_); } private: QLineEdit *name_; }; // --- Page 2: start URL, depth, host/pattern filters ------------------------------------- class GrabberStartPage : public QWizardPage { public: explicit GrabberStartPage(QWidget *parent) : QWizardPage(parent) { setTitle(QObject::tr("Start URL")); startUrl_ = new QLineEdit(this); startUrl_->setPlaceholderText(QStringLiteral("https://example.com/gallery/")); registerField(QStringLiteral("startUrl*"), startUrl_); depth_ = new QSpinBox(this); depth_->setRange(0, 10); depth_->setValue(1); sameHostOnly_ = new QCheckBox(QObject::tr("Stay on the same host"), this); sameHostOnly_->setChecked(true); includePatterns_ = new QLineEdit(this); includePatterns_->setPlaceholderText(QObject::tr("comma-separated globs, optional")); excludePatterns_ = new QLineEdit(this); excludePatterns_->setPlaceholderText(QObject::tr("comma-separated globs, optional")); auto *layout = new QFormLayout(this); layout->addRow(QObject::tr("Start URL:"), startUrl_); layout->addRow(QObject::tr("Crawl depth:"), depth_); layout->addRow(sameHostOnly_); layout->addRow(QObject::tr("Include only:"), includePatterns_); layout->addRow(QObject::tr("Exclude:"), excludePatterns_); } QString startUrl() const { return startUrl_->text().trimmed(); } int depth() const { return depth_->value(); } bool sameHostOnly() const { return sameHostOnly_->isChecked(); } QString includePatterns() const { return includePatterns_->text(); } QString excludePatterns() const { return excludePatterns_->text(); } private: QLineEdit *startUrl_; QSpinBox *depth_; QCheckBox *sameHostOnly_; QLineEdit *includePatterns_; QLineEdit *excludePatterns_; }; // --- Page 3: file-type filter ------------------------------------------------------------ class GrabberFileTypesPage : public QWizardPage { public: explicit GrabberFileTypesPage(QWidget *parent) : QWizardPage(parent) { setTitle(QObject::tr("File types")); setSubTitle(QObject::tr("Leave everything unchecked to crawl every file type.")); images_ = new QCheckBox(QObject::tr("Images (jpg, png, gif, webp)"), this); archives_ = new QCheckBox(QObject::tr("Archives (zip, rar, 7z, tar, gz)"), this); docs_ = new QCheckBox(QObject::tr("Documents (pdf, doc, docx, xls, xlsx)"), this); video_ = new QCheckBox(QObject::tr("Video (mp4, mkv, avi, webm)"), this); audio_ = new QCheckBox(QObject::tr("Audio (mp3, flac, wav, ogg)"), this); custom_ = new QLineEdit(this); custom_->setPlaceholderText(QObject::tr("custom extensions, comma-separated")); auto *layout = new QVBoxLayout(this); layout->addWidget(images_); layout->addWidget(archives_); layout->addWidget(docs_); layout->addWidget(video_); layout->addWidget(audio_); auto *customRow = new QFormLayout; customRow->addRow(QObject::tr("Also:"), custom_); layout->addLayout(customRow); } bool images() const { return images_->isChecked(); } bool archives() const { return archives_->isChecked(); } bool docs() const { return docs_->isChecked(); } bool video() const { return video_->isChecked(); } bool audio() const { return audio_->isChecked(); } QString customCsv() const { return custom_->text(); } private: QCheckBox *images_; QCheckBox *archives_; QCheckBox *docs_; QCheckBox *video_; QCheckBox *audio_; QLineEdit *custom_; }; // --- Page 4: crawl + review + harvest ----------------------------------------------------- class GrabberReviewPage : public QWizardPage { Q_OBJECT public: GrabberReviewPage(rpc::RpcClient *client, QJsonArray categories, GrabberStartPage *startPage, GrabberFileTypesPage *fileTypesPage, QWidget *parent) : QWizardPage(parent), client_(client), startPage_(startPage), fileTypesPage_(fileTypesPage) { setTitle(QObject::tr("Review found files")); setFinalPage(true); statusLabel_ = new QLabel(this); table_ = new QTableWidget(0, 3, this); table_->setHorizontalHeaderLabels( {QObject::tr("Get"), QObject::tr("File"), QObject::tr("Size")}); table_->horizontalHeader()->setSectionResizeMode(1, QHeaderView::Stretch); table_->verticalHeader()->setVisible(false); table_->setEditTriggers(QAbstractItemView::NoEditTriggers); auto *selectAll = new QPushButton(QObject::tr("Select All"), this); auto *selectNone = new QPushButton(QObject::tr("Select None"), this); connect(selectAll, &QPushButton::clicked, this, [this] { setAllChecked(true); }); connect(selectNone, &QPushButton::clicked, this, [this] { setAllChecked(false); }); auto *selectRow = new QHBoxLayout; selectRow->addWidget(selectAll); selectRow->addWidget(selectNone); selectRow->addStretch(); categoryCombo_ = new QComboBox(this); categoryCombo_->addItem(QObject::tr("(Automatic)"), QString()); for (const QJsonValue &v : categories) { const QJsonObject c = v.toObject(); categoryCombo_->addItem(c.value("name").toString(), c.value("categoryId").toString()); } startModeCombo_ = new QComboBox(this); startModeCombo_->addItem(QObject::tr("Download Now"), QStringLiteral("now")); startModeCombo_->addItem(QObject::tr("Download Later"), QStringLiteral("later")); errorLabel_ = new QLabel(this); errorLabel_->setStyleSheet(QStringLiteral("color: #c0392b;")); errorLabel_->hide(); auto *footer = new QFormLayout; footer->addRow(QObject::tr("Category:"), categoryCombo_); footer->addRow(QObject::tr("Start mode:"), startModeCombo_); auto *layout = new QVBoxLayout(this); layout->addWidget(statusLabel_); layout->addLayout(selectRow); layout->addWidget(table_, 1); layout->addLayout(footer); layout->addWidget(errorLabel_); pollTimer_ = new QTimer(this); pollTimer_->setInterval(5000); // catch-up fallback; live updates come from the event connect(pollTimer_, &QTimer::timeout, this, &GrabberReviewPage::fetchStatus); connect(client_, &rpc::RpcClient::grabberProgress, this, &GrabberReviewPage::onProgress); } void initializePage() override { if (started_) { return; } started_ = true; statusLabel_->setText(QObject::tr("Starting crawl…")); const QJsonValue fileTypes = GrabberWizard::buildFileTypes( fileTypesPage_->images(), fileTypesPage_->archives(), fileTypesPage_->docs(), fileTypesPage_->video(), fileTypesPage_->audio(), fileTypesPage_->customCsv()); QJsonObject params = GrabberWizard::buildStartParams( startPage_->startUrl(), startPage_->depth(), startPage_->sameHostOnly(), startPage_->includePatterns(), startPage_->excludePatterns(), fileTypes); QPointer self(this); client_->call( QString::fromLatin1(rpc::method::kGrabberStart), params, [self](const rpc::RpcReply &reply) { if (!self) { return; } if (!reply.ok()) { self->statusLabel_->setText( QObject::tr("Could not start the crawl: %1").arg(reply.error.message)); return; } self->jobId_ = reply.result.toObject().value("jobId").toString(); self->pollTimer_->start(); self->fetchStatus(); }); } bool isComplete() const override { return !files_.isEmpty(); } /// QWizard calls this on Finish and only advances (closes, for the last page) when it /// returns true. harvest is async, so this always returns false and closes the wizard /// itself from the reply callback instead — the same submit-then-accept shape every /// other dialog here uses, just routed through validatePage() rather than a button. bool validatePage() override { if (harvesting_) { return false; } QStringList selected; for (int row = 0; row < table_->rowCount(); ++row) { if (table_->item(row, 0)->checkState() == Qt::Checked) { selected << table_->item(row, 0)->data(Qt::UserRole).toString(); } } if (selected.isEmpty()) { return false; } harvesting_ = true; const QJsonObject params = GrabberWizard::buildHarvestParams( jobId_, selected, categoryCombo_->currentData().toString(), startModeCombo_->currentData().toString()); QPointer self(this); client_->call( QString::fromLatin1(rpc::method::kGrabberHarvest), params, [self](const rpc::RpcReply &reply) { if (!self) { return; } self->harvesting_ = false; if (!reply.ok()) { self->errorLabel_->setText(QObject::tr("Could not add the selected files: %1") .arg(reply.error.message)); self->errorLabel_->show(); return; } self->wizard()->accept(); }); return false; } private: void setAllChecked(bool checked) { for (int row = 0; row < table_->rowCount(); ++row) { table_->item(row, 0)->setCheckState(checked ? Qt::Checked : Qt::Unchecked); } } void onProgress(const QJsonObject ¶ms) { if (params.value("jobId").toString() != jobId_ || jobId_.isEmpty()) { return; } statusLabel_->setText(QObject::tr("Crawled %1, found %2…") .arg(params.value("crawled").toInt()) .arg(params.value("found").toInt())); if (params.value("done").toBool()) { fetchStatus(); } } void fetchStatus() { if (jobId_.isEmpty()) { return; } QPointer self(this); client_->call(QString::fromLatin1(rpc::method::kGrabberStatus), QJsonObject{{"jobId", jobId_}}, [self](const rpc::RpcReply &reply) { if (!self || !reply.ok()) { return; } self->applyStatus(reply.result.toObject()); }); } void applyStatus(const QJsonObject &status) { const QString state = status.value("state").toString(); files_ = status.value("files").toArray(); populateTable(); if (state == QLatin1String("crawling")) { statusLabel_->setText(QObject::tr("Crawling… %1 found so far").arg(files_.size())); } else if (state == QLatin1String("done")) { pollTimer_->stop(); statusLabel_->setText(QObject::tr("Done — %1 file(s) found.").arg(files_.size())); } else { pollTimer_->stop(); statusLabel_->setText( QObject::tr("Crawl %1: %2").arg(state, status.value("error").toString())); } emit completeChanged(); } void populateTable() { table_->setRowCount(files_.size()); for (int row = 0; row < files_.size(); ++row) { const QJsonObject f = files_[row].toObject(); auto *checkItem = new QTableWidgetItem(f.value("filename").toString(f.value("url").toString())); checkItem->setFlags(checkItem->flags() | Qt::ItemIsUserCheckable); checkItem->setCheckState(Qt::Checked); checkItem->setData(Qt::UserRole, f.value("fileId").toString()); table_->setItem(row, 0, checkItem); table_->setItem(row, 1, new QTableWidgetItem(f.value("url").toString())); const QJsonValue size = f.value("sizeBytes"); table_->setItem( row, 2, new QTableWidgetItem(size.isDouble() ? QLocale().formattedDataSize( static_cast(size.toDouble())) : QObject::tr("unknown"))); } } rpc::RpcClient *client_; GrabberStartPage *startPage_; GrabberFileTypesPage *fileTypesPage_; QString jobId_; QJsonArray files_; bool started_ = false; bool harvesting_ = false; QLabel *statusLabel_; QTableWidget *table_; QComboBox *categoryCombo_; QComboBox *startModeCombo_; QLabel *errorLabel_; QTimer *pollTimer_; }; } // namespace QJsonValue GrabberWizard::buildFileTypes(bool images, bool archives, bool docs, bool video, bool audio, const QString &customCsv) { QJsonArray types; if (images) { for (const char *e : {"jpg", "jpeg", "png", "gif", "webp"}) { types.append(e); } } if (archives) { for (const char *e : {"zip", "rar", "7z", "tar", "gz"}) { types.append(e); } } if (docs) { for (const char *e : {"pdf", "doc", "docx", "xls", "xlsx"}) { types.append(e); } } if (video) { for (const char *e : {"mp4", "mkv", "avi", "webm"}) { types.append(e); } } if (audio) { for (const char *e : {"mp3", "flac", "wav", "ogg"}) { types.append(e); } } for (const QString &e : splitList(customCsv)) { types.append(e); } return types.isEmpty() ? QJsonValue(QJsonValue::Null) : QJsonValue(types); } QJsonObject GrabberWizard::buildStartParams(const QString &startUrl, int depth, bool sameHostOnly, const QString &includePatterns, const QString &excludePatterns, const QJsonValue &fileTypes) { const QStringList include = splitList(includePatterns); const QStringList exclude = splitList(excludePatterns); return QJsonObject{ {"startUrl", startUrl}, {"depth", depth}, {"sameHostOnly", sameHostOnly}, {"includePatterns", include.isEmpty() ? QJsonValue(QJsonValue::Null) : QJsonValue(QJsonArray::fromStringList(include))}, {"excludePatterns", exclude.isEmpty() ? QJsonValue(QJsonValue::Null) : QJsonValue(QJsonArray::fromStringList(exclude))}, {"fileTypes", fileTypes}, }; } QJsonObject GrabberWizard::buildHarvestParams(const QString &jobId, const QStringList &selectedIds, const QString &categoryId, const QString &startMode) { QJsonObject defaults{{"url", QString()}, {"startMode", startMode}}; // url is ignored here if (!categoryId.isEmpty()) { defaults["categoryId"] = categoryId; } return QJsonObject{ {"jobId", jobId}, {"select", QJsonArray::fromStringList(selectedIds)}, {"defaults", defaults}, }; } GrabberWizard::GrabberWizard(rpc::RpcClient *client, QJsonArray categories, QWidget *parent) : QWizard(parent), client_(client), categories_(std::move(categories)) { setWindowTitle(tr("Site Grabber")); resize(640, 520); addPage(new GrabberProjectPage(this)); auto *startPage = new GrabberStartPage(this); addPage(startPage); auto *fileTypesPage = new GrabberFileTypesPage(this); addPage(fileTypesPage); addPage(new GrabberReviewPage(client_, categories_, startPage, fileTypesPage, this)); } } // namespace velox::gui #include "GrabberWizard.moc"