diff --git a/src/gui/src/settings/options-window.cpp b/src/gui/src/settings/options-window.cpp index 89fe1cddf..2331a4776 100644 --- a/src/gui/src/settings/options-window.cpp +++ b/src/gui/src/settings/options-window.cpp @@ -153,6 +153,9 @@ OptionsWindow::OptionsWindow(Profile *profile, ThemeLoader *themeLoader, QWidget }); ffmpegVersionWatcher->setFuture(ffmpegVersion); ui->checkConversionFFmpegRemuxWebmToMp4->setChecked(settings->value("Save/FFmpegRemuxWebmToMp4", false).toBool()); + ui->checkConversionUgoiraEnabled->setChecked(settings->value("Save/ConvertUgoira", false).toBool()); + ui->comboConversionUgoiraTargetExtension->setCurrentText(settings->value("Save/ConvertUgoiraFormat", "gif").toString().toUpper()); + ui->checkConversionUgoiraDelete->setChecked(settings->value("Save/ConvertUgoiraDeleteOriginal", false).toBool()); // Metadata using Windows Property System #ifndef WIN_FILE_PROPS @@ -1274,6 +1277,9 @@ void OptionsWindow::save() } settings->setValue("FFmpegRemuxWebmToMp4", ui->checkConversionFFmpegRemuxWebmToMp4->isChecked()); + settings->setValue("ConvertUgoira", ui->checkConversionUgoiraEnabled->isChecked()); + settings->setValue("ConvertUgoiraFormat", ui->comboConversionUgoiraTargetExtension->currentText().toLower()); + settings->setValue("ConvertUgoiraDeleteOriginal", ui->checkConversionUgoiraDelete->isChecked()); settings->setValue("MetadataPropsysExtensions", ui->lineMetadataPropsysExtensions->text()); settings->beginWriteArray("MetadataPropsys"); diff --git a/src/gui/src/settings/options-window.ui b/src/gui/src/settings/options-window.ui index 2c4e03558..84c45a17e 100644 --- a/src/gui/src/settings/options-window.ui +++ b/src/gui/src/settings/options-window.ui @@ -6,8 +6,8 @@ 0 0 - 747 - 628 + 743 + 619 @@ -1181,8 +1181,8 @@ 0 0 - 476 - 475 + 473 + 466 @@ -1463,6 +1463,56 @@ + + + + Target format + + + + + + + + GIF + + + + + WEBM + + + + + APNG + + + + + WEBP + + + + + MKV + + + + + + + + Delete original ugoira ZIP files + + + + + + + Convert ugoira ZIP files + + + @@ -5996,8 +6046,8 @@ addLogFile() - 336 - 40 + 584 + 97 659 @@ -6044,8 +6094,8 @@ addSourceRegistry() - 336 - 40 + 584 + 81 738 @@ -6080,7 +6130,7 @@ 40 - 749 + 742 79 @@ -6096,8 +6146,8 @@ 263 - 718 - 325 + 720 + 308 @@ -6108,12 +6158,44 @@ setDisabled(bool) - 523 - 507 + 336 + 40 + + + 336 + 40 + + + + + checkConversionUgoiraEnabled + toggled(bool) + comboConversionUgoiraTargetExtension + setEnabled(bool) + + + 371 + 113 + + + 458 + 138 + + + + + checkConversionUgoiraEnabled + toggled(bool) + checkConversionUgoiraDelete + setEnabled(bool) + + + 322 + 114 - 515 - 522 + 324 + 171 diff --git a/src/lib/src/downloader/image-downloader.cpp b/src/lib/src/downloader/image-downloader.cpp index 7168f4229..6437d0734 100644 --- a/src/lib/src/downloader/image-downloader.cpp +++ b/src/lib/src/downloader/image-downloader.cpp @@ -81,6 +81,7 @@ void ImageDownloader::save() // Always load details if the API doesn't provide the file URL in the listing page const QStringList forcedTokens = m_image->parentSite()->getApis().first()->forcedTokens(); const bool needFileUrl = forcedTokens.contains("*") || forcedTokens.contains("file_url"); + const bool needUgoiraData = m_image->extension() == QStringLiteral("zip") && m_profile->getSettings()->value("Save/ConvertUgoira", false).toBool(); // If we use direct saving or don't want to load tags, we directly save the image const int globalNeedTags = needExactTags(m_profile->getSettings()); @@ -88,7 +89,7 @@ void ImageDownloader::save() const int needTags = qMax(globalNeedTags, localNeedTags); const bool filenameNeedTags = needTags == 2 || (needTags == 1 && m_image->hasUnknownTag()); const bool blacklistNeedTags = m_blacklist != nullptr && !m_blacklist->isEmpty() && m_image->tags().isEmpty(); - if (!blacklistNeedTags && !needFileUrl && (!m_loadTags || !m_paths.isEmpty() || !filenameNeedTags)) { + if (!blacklistNeedTags && !needFileUrl && !needUgoiraData && (!m_loadTags || !m_paths.isEmpty() || !filenameNeedTags)) { loadedSave(Image::LoadTagsResult::Ok); return; } diff --git a/src/lib/src/ffmpeg.cpp b/src/lib/src/ffmpeg.cpp index ee7d0efbb..ad4efdbdd 100644 --- a/src/lib/src/ffmpeg.cpp +++ b/src/lib/src/ffmpeg.cpp @@ -1,8 +1,12 @@ #include "ffmpeg.h" #include +#include #include #include +#include +#include "functions.h" #include "logger.h" +#include "utils/zip.h" QString FFmpeg::version(int msecs) @@ -41,7 +45,7 @@ QString FFmpeg::remux(const QString &file, const QString &extension, bool delete { // Since the method takes an extension, build an absolute path to the input file with that extension const QFileInfo info(file); - const QString destination = info.path() + QDir::separator() + info.completeBaseName() + "." + extension; + QString destination = info.path() + QDir::separator() + info.completeBaseName() + "." + extension; // Ensure the operation is safe to do if (!QFile::exists(file)) { @@ -53,13 +57,113 @@ QString FFmpeg::remux(const QString &file, const QString &extension, bool delete return file; } + // Execute the conversion command + const QStringList params = { "-n", "-loglevel", "error", "-i", file, "-c", "copy", destination }; + if (!execute(params, msecs)) { + return file; + } + + // Copy file creation information + setFileCreationDate(destination, info.lastModified()); + + // On success, delete the original file if requested + if (deleteOriginal) { + QFile::remove(file); + } + + return destination; +} + +QString FFmpeg::convertUgoira(const QString &file, const QList> &frameInformation, const QString &extension, bool deleteOriginal, int msecs) +{ + // Since the method takes an extension, build an absolute path to the input file with that extension + const QFileInfo info(file); + QString destination = info.path() + QDir::separator() + info.completeBaseName() + "." + extension; + + // Ensure the operation is safe to do + if (info.suffix() != QStringLiteral("zip")) { + log(QStringLiteral("Cannot convert ugoira file that is not a ZIP: `%1`").arg(file), Logger::Error); + return file; + } + if (!QFile::exists(file)) { + log(QStringLiteral("Cannot convert ugoira file that does not exist: `%1`").arg(file), Logger::Error); + return file; + } + if (QFile::exists(destination)) { + log(QStringLiteral("Converting the ugoira file `%1` would overwrite another file: `%2`").arg(file, destination), Logger::Error); + return file; + } + + // Extract the ugoira ZIP file + QTemporaryDir tmpDir; + const QString tmpDirPath = tmpDir.path(); + if (!tmpDir.isValid() || !unzipFile(file, tmpDir.path())) { + log(QStringLiteral("Could not extract ugoira ZIP file `%1` into directory: `%2`").arg(file, destination), Logger::Error); + return file; + } + + // List all frame files from the ZIP + QStringList frameFiles = QDir(tmpDir.path()).entryList(QDir::Files | QDir::NoDotAndDotDot); + if (frameInformation.count() != frameFiles.count()) { + log(QStringLiteral("Could not extract ugoira ZIP file `%1` into directory: `%2`").arg(file, destination), Logger::Error); + return file; + } + + // Build the ffmpeg concatenation string + QFile ffconcatFile(tmpDir.filePath("ffconcat.txt")); + if (!ffconcatFile.open(QFile::WriteOnly)) { + log(QStringLiteral("Could not create temporary ffconcat file: `%1`").arg(ffconcatFile.fileName()), Logger::Error); + return file; + } + QString ffconcat = "ffconcat version 1.0\n"; + for (const auto &frame : frameInformation) { + ffconcat += "file " + (frame.first.isEmpty() ? frameFiles.takeFirst() : frame.first) + '\n'; + ffconcat += "duration " + QString::number(float(frame.second) / 1000) + '\n'; + } + ffconcatFile.write(ffconcat.toUtf8()); + ffconcatFile.close(); + + // Build the params + QStringList params = { "-n", "-loglevel", "error", "-i", ffconcatFile.fileName() }; + if (extension == QStringLiteral("gif")) { + params.append({ "-filter_complex", "[0:v]split[a][b];[a]palettegen=stats_mode=diff[p];[b][p]paletteuse=dither=bayer:bayer_scale=5:diff_mode=rectangle", "-vsync", "0" }); + } else if (extension == QStringLiteral("apng")) { + params.append({ "-c:v", "apng", "-plays", "0", "-vsync", "0" }); + } else if (extension == QStringLiteral("webp")) { + params.append({ "-c:v", "libwebp", "-lossless", "0", "-compression_level", "5", "-quality", "100", "-loop", "0", "-vsync", "0" }); + } else if (extension == QStringLiteral("webm")) { + params.append({ "-c:v", "libvpx-vp9", "-lossless", "0", "-crf", "15", "-b", "0", "-vsync", "0" }); + } else { + params.append({ "-c:v", "copy" }); + } + params.append(destination); + + // Execute the conversion command + if (!execute(params, msecs)) { + return file; + } + + // Copy file creation information + setFileCreationDate(destination, info.lastModified()); + + // On success, delete the original file if requested + if (deleteOriginal) { + QFile::remove(file); + } + + return destination; +} + + +bool FFmpeg::execute(const QStringList ¶ms, int msecs) +{ QProcess process; - process.start("ffmpeg", { "-n", "-loglevel", "error", "-i", file, "-c", "copy", destination }); + process.start("ffmpeg", params); // Ensure the process started successfully if (!process.waitForStarted(msecs)) { log(QStringLiteral("Could not start FFmpeg")); - return file; + return false; } // Wait for FFmpeg to finish @@ -75,10 +179,5 @@ QString FFmpeg::remux(const QString &file, const QString &extension, bool delete log(QString("[Exiftool] %1").arg(standardError), Logger::Error); } - // On success, delete the original file if requested - if (ok && deleteOriginal) { - QFile::remove(file); - } - - return destination; + return ok; } diff --git a/src/lib/src/ffmpeg.h b/src/lib/src/ffmpeg.h index 2f923ad6c..42bb2d1c3 100644 --- a/src/lib/src/ffmpeg.h +++ b/src/lib/src/ffmpeg.h @@ -25,6 +25,21 @@ class FFmpeg * @return The destination file path on success, the original file path on error. */ static QString remux(const QString &file, const QString &extension, bool deleteOriginal = true, int msecs = 30000); + + /** + * Convert a ugoira ZIP file to a different format. + * + * @param file The file to remux. + * @param frameInformation A list of (frameFile, delay) tuples representing each frame in this ugoira ZIP file. + * @param extension The target extension (ex: "gif"). + * @param deleteOriginal Whether to delete the original file on success. + * @param msecs The duration to wait in milliseconds for the command to run. + * @return The destination file path on success if the original was deleted, the original file path otherwise. + */ + static QString convertUgoira(const QString &file, const QList> &frameInformation, const QString &extension, bool deleteOriginal = true, int msecs = 30000); + + protected: + static bool execute(const QStringList ¶ms, int msecs = 30000); }; #endif // FFMPEG_H diff --git a/src/lib/src/models/api/api-endpoint.h b/src/lib/src/models/api/api-endpoint.h index a8cebdfeb..818985d0e 100644 --- a/src/lib/src/models/api/api-endpoint.h +++ b/src/lib/src/models/api/api-endpoint.h @@ -19,6 +19,7 @@ class ApiEndpoint virtual PageUrl url(const QMap &query, int page, int limit, const PageInformation &lastPage, Site *site) const = 0; virtual ParsedPage parse(Page *parentPage, const QString &source, int statusCode, int first) const = 0; + virtual QVariant parseAny(const QString &source, int statusCode) const = 0; }; #endif // API_ENDPOINT_H diff --git a/src/lib/src/models/api/javascript-api-endpoint.cpp b/src/lib/src/models/api/javascript-api-endpoint.cpp index ae4f2378f..6b049ded5 100644 --- a/src/lib/src/models/api/javascript-api-endpoint.cpp +++ b/src/lib/src/models/api/javascript-api-endpoint.cpp @@ -109,3 +109,10 @@ ParsedPage JavascriptApiEndpoint::parse(Page *parentPage, const QString &source, return ret; } + +QVariant JavascriptApiEndpoint::parseAny(const QString &source, int statusCode) const +{ + const QJSValue parseFunction = m_endpoint.property("parse"); + const QJSValue &result = parseFunction.call(QList { source, statusCode }); + return result.toVariant(); +} diff --git a/src/lib/src/models/api/javascript-api-endpoint.h b/src/lib/src/models/api/javascript-api-endpoint.h index 91363e8f0..b68d36689 100644 --- a/src/lib/src/models/api/javascript-api-endpoint.h +++ b/src/lib/src/models/api/javascript-api-endpoint.h @@ -25,6 +25,7 @@ class JavascriptApiEndpoint : public QObject, public ApiEndpoint PageUrl url(const QMap &query, int page, int limit, const PageInformation &lastPage, Site *site) const override; ParsedPage parse(Page *parentPage, const QString &source, int statusCode, int first) const override; + QVariant parseAny(const QString &source, int statusCode) const override; private: JavascriptApi *m_api; // TODO: get rid of this diff --git a/src/lib/src/models/image.cpp b/src/lib/src/models/image.cpp index 985dee8d1..ea9d31e4e 100644 --- a/src/lib/src/models/image.cpp +++ b/src/lib/src/models/image.cpp @@ -585,6 +585,44 @@ void Image::parseDetails() refreshTokens(); + // If we load the details for an ugoira file that we will want to convert later, load the ugoira metadata as well + if (extension() == QStringLiteral("zip") && m_settings->value("Save/ConvertUgoira", false).toBool()) { + auto *endpoint = m_parentSite->apiEndpoint("ugoira_details"); + if (endpoint != nullptr) { + const QString ugoiraDetailsUrl = endpoint->url(m_identity, 1, 1, {}, m_parentSite).url; + + log(QStringLiteral("Loading image ugoira details from `%1`").arg(ugoiraDetailsUrl), Logger::Info); + auto *reply = m_parentSite->get(ugoiraDetailsUrl, Site::QueryType::Details); + reply->setParent(this); + + connect(reply, &NetworkReply::finished, this, &Image::parseUgoiraDetails); + return; + } + } + + emit finishedLoadingTags(LoadTagsResult::Ok); +} +void Image::parseUgoiraDetails() +{ + auto *reply = qobject_cast(sender()); + auto *endpoint = m_parentSite->apiEndpoint("ugoira_details"); + + // Handle network errors + if (reply->error()) { + if (reply->error() != NetworkReply::NetworkError::OperationCanceledError) { + log(QStringLiteral("Loading ugoira details error for '%1': %2").arg(reply->url().toString(), reply->errorString()), Logger::Error); + } + reply->deleteLater(); + emit finishedLoadingTags(LoadTagsResult::NetworkError); + return; + } + + // Parse the metadata + const QString source = QString::fromUtf8(reply->readAll()); + const int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + m_data["ugoira_metadata"] = endpoint->parseAny(source, statusCode); + + reply->deleteLater(); emit finishedLoadingTags(LoadTagsResult::Ok); } @@ -774,6 +812,13 @@ QString Image::postSaving(const QString &originalPath, Size size, bool addMd5, b path = FFmpeg::remux(path, "mp4"); } + // Ugoira conversion + if (ext == QStringLiteral("zip") && m_settings->value("Save/ConvertUgoira", false).toBool()) { + const QString targetExt = m_settings->value("Save/ConvertUgoiraFormat", "gif").toString(); + const bool deleteOriginal = m_settings->value("Save/ConvertUgoiraDeleteOriginal", false).toBool(); + path = FFmpeg::convertUgoira(path, ugoiraFrameInformation(), targetExt, deleteOriginal); + } + // Metadata #ifdef WIN_FILE_PROPS const QStringList exts = m_settings->value("Save/MetadataPropsysExtensions", "jpg jpeg mp4").toString().split(' ', Qt::SkipEmptyParts); @@ -1265,3 +1310,23 @@ const ImageSize &Image::mediaForSize(const QSize &size) return *ret; } + +QList> Image::ugoiraFrameInformation() const +{ + // Ensure the ugoira metadata is loaded first + const QVariant ugoiraMetadata = m_data.value("ugoira_metadata"); + if (!ugoiraMetadata.isValid() || ugoiraMetadata.isNull()) { + return {}; + } + + QList> frameInformation; + + const auto frames = ugoiraMetadata.toMap()["frames"].toList(); + for (const QVariant &frame : frames) { + const auto obj = frame.toMap(); + const QString file = obj["file"].isNull() ? "" : obj["file"].toString(); + frameInformation.append({file, obj["delay"].toInt()}); + } + + return frameInformation; +} diff --git a/src/lib/src/models/image.h b/src/lib/src/models/image.h index 2a178072c..f7738ba3f 100644 --- a/src/lib/src/models/image.h +++ b/src/lib/src/models/image.h @@ -90,6 +90,7 @@ class Image : public QObject, public Downloadable bool isValid() const; Profile *getProfile() const { return m_profile; } const ImageSize &mediaForSize(const QSize &size); + QList> ugoiraFrameInformation() const; // Preview pixmap store QPixmap previewImage() const; @@ -131,6 +132,7 @@ class Image : public QObject, public Downloadable void loadDetails(bool rateLimit = false); void abortTags(); void parseDetails(); + void parseUgoiraDetails(); signals: void finishedLoadingPreview(); diff --git a/src/lib/src/models/site.cpp b/src/lib/src/models/site.cpp index 30dbbfd18..fbfcfe81a 100644 --- a/src/lib/src/models/site.cpp +++ b/src/lib/src/models/site.cpp @@ -402,6 +402,16 @@ Api *Site::tagsApi() const } return nullptr; } +ApiEndpoint *Site::apiEndpoint(const QString &name) const +{ + for (Api *api : m_apis) { + ApiEndpoint *endpoint = api->endpoints().value(name); + if (endpoint != nullptr) { + return endpoint; + } + } + return nullptr; +} bool Site::autoLogin() const { return m_autoLogin; } void Site::setAutoLogin(bool autoLogin) { m_autoLogin = autoLogin; } diff --git a/src/lib/src/models/site.h b/src/lib/src/models/site.h index 87b9142b2..22357e2c3 100644 --- a/src/lib/src/models/site.h +++ b/src/lib/src/models/site.h @@ -9,6 +9,7 @@ class Api; +class ApiEndpoint; class Auth; class Image; class MixedSettings; @@ -89,6 +90,7 @@ class Site : public QObject Api *detailsApi() const; Api *fullDetailsApi() const; Api *tagsApi() const; + ApiEndpoint *apiEndpoint(const QString &name) const; // Login void setAutoLogin(bool autoLogin); diff --git a/src/sites/Danbooru (2.0)/model.ts b/src/sites/Danbooru (2.0)/model.ts index 527e9e265..ecf3c7a28 100644 --- a/src/sites/Danbooru (2.0)/model.ts +++ b/src/sites/Danbooru (2.0)/model.ts @@ -1,4 +1,9 @@ function completeImage(img: IImage): IImage { + img.identity = { + "id": img.id!, + "md5": img.md5!, + }; + if (img.ext && img.ext[0] === ".") { img.ext = img.ext.substr(1); } @@ -155,6 +160,22 @@ export const source: ISource = { }, }, endpoints: { + ugoira_details: { + input: { + id: { + type: "input", + }, + }, + url: (query: Record<"id", number>): string => { + return "/posts/" + String(query.id) + ".json?only=media_metadata"; + }, + parse: (src: string): IParsedUgoiraDetails => { + const delays = JSON.parse(src)["media_metadata"]["metadata"]["Ugoira:FrameDelays"]; + return { + frames: delays.map((delay: number) => ({ delay })), + }; + }, + }, pool_list: { name: "Pools", input: { diff --git a/src/sites/Pixiv/model.ts b/src/sites/Pixiv/model.ts index 30b55116a..bbea5462f 100644 --- a/src/sites/Pixiv/model.ts +++ b/src/sites/Pixiv/model.ts @@ -94,6 +94,7 @@ function parseImage(image: any, fromGallery: boolean): IImage { }; const img = Grabber.mapFields(image, map); + img.identity = {"id": img["id"]}; if (image["age_limit"] === "all-age") { img.rating = "safe"; } else if (image["age_limit"] === "r18") { @@ -286,6 +287,21 @@ export const source: ISource = { return img; }, }, + endpoints: { + ugoira_details: { + input: { + id: { + type: "input", + }, + }, + url: (query: Record<"id", number>): string => { + return "https://app-api.pixiv.net/v1/ugoira/metadata?illust_id=" + String(query.id); + }, + parse: (src: string): IParsedUgoiraDetails => { + return JSON.parse(src)["ugoira_metadata"]; + }, + } + } }, }, }; diff --git a/src/sites/types.d.ts b/src/sites/types.d.ts index a5b19e75f..0a33f7352 100644 --- a/src/sites/types.d.ts +++ b/src/sites/types.d.ts @@ -727,6 +727,23 @@ interface IUrlOptions extends IUrlOptionsBase { limit: number; } +interface IParsedUgoiraDetails { + /** + * The list of all frames contained in this ugoira file, sorted in order. + */ + frames: { + /** + * The file for that frame. If empty, will take the next frame from the ZIP file alphabetically. + */ + file?: string; + + /** + * The duration for which this frame should be shown. + */ + delay: number; + }[]; +} + /** * Additional information for a details query. */ @@ -802,7 +819,7 @@ interface IEndpoint { /** * The function that will parse the response of the URL above. */ - parse: (src: string, statusCode: number) => IParsedSearch | IError; + parse: (src: string, statusCode: number) => IParsedSearch | IParsedUgoiraDetails | IError; } /**