From 0e2799f728f5160f4c00a3318cd2bbf780df5193 Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Sun, 30 Jul 2023 11:02:10 +0200 Subject: [PATCH 1/8] Rename functions --- .../server/util/WebInterfaceManager.kt | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt index ca86af92ab..685abce4f7 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt @@ -149,7 +149,7 @@ object WebInterfaceManager { */ val doDownload = { try { - downloadLatestCompatibleVersion() + downloadVersion() } catch (e: Exception) { false } || isLocalWebUIValid @@ -205,7 +205,7 @@ object WebInterfaceManager { } logger.info { "checkForUpdate(${serverConfig.webUIFlavor}, $localVersion): An update is available, starting download..." } - downloadLatestCompatibleVersion() + downloadVersion() } private fun getDownloadUrlFor(version: String): String { @@ -311,28 +311,28 @@ object WebInterfaceManager { throw Exception("No compatible webUI version found") } - fun downloadLatestCompatibleVersion(retryCount: Int = 0): Boolean { + fun downloadVersion(retryCount: Int = 0): Boolean { val latestCompatibleVersion = getLatestCompatibleVersion() val webUIZip = "${WebUI.WEBUI.baseFileName}-$latestCompatibleVersion.zip" val webUIZipPath = "$tmpDir/$webUIZip" val webUIZipFile = File(webUIZipPath) - logger.info { "downloadLatestCompatibleVersion: Downloading WebUI (flavor= ${serverConfig.webUIFlavor}, version \"$latestCompatibleVersion\") zip from the Internet..." } + logger.info { "downloadVersion: Downloading WebUI (flavor= ${serverConfig.webUIFlavor}, version \"$latestCompatibleVersion\") zip from the Internet..." } try { val webUIZipURL = "${getDownloadUrlFor(latestCompatibleVersion)}/$webUIZip" - downloadVersion(webUIZipURL, webUIZipFile) + downloadVersionZipFile(webUIZipURL, webUIZipFile) if (!isDownloadValid(webUIZip, webUIZipPath)) { throw Exception("Download is invalid") } } catch (e: Exception) { val retry = retryCount < 3 - logger.error { "downloadLatestCompatibleVersion: Download failed${if (retry) ", retrying ${retryCount + 1}/3" else ""} - error: $e" } + logger.error { "downloadVersion: Download failed${if (retry) ", retrying ${retryCount + 1}/3" else ""} - error: $e" } if (retry) { - return downloadLatestCompatibleVersion(retryCount + 1) + return downloadVersion(retryCount + 1) } return false @@ -341,14 +341,14 @@ object WebInterfaceManager { File(applicationDirs.webUIRoot).deleteRecursively() // extract webUI zip - logger.info { "downloadLatestCompatibleVersion: Extracting WebUI zip..." } + logger.info { "downloadVersion: Extracting WebUI zip..." } extractDownload(webUIZipPath, applicationDirs.webUIRoot) - logger.info { "downloadLatestCompatibleVersion: Extracting WebUI zip Done." } + logger.info { "downloadVersion: Extracting WebUI zip Done." } return true } - private fun downloadVersion(url: String, zipFile: File) { + private fun downloadVersionZipFile(url: String, zipFile: File) { zipFile.delete() val data = ByteArray(1024) @@ -361,7 +361,7 @@ object WebInterfaceManager { connection.inputStream.buffered().use { inp -> var totalCount = 0 - print("downloadVersion: Download progress: % 00") + print("downloadVersionZipFile: Download progress: % 00") while (true) { val count = inp.read(data, 0, 1024) @@ -377,7 +377,7 @@ object WebInterfaceManager { webUIZipFileOut.write(data, 0, count) } println() - logger.info { "downloadVersion: Downloading WebUI Done." } + logger.info { "downloadVersionZipFile: Downloading WebUI Done." } } } } From ac18deb0bd83287113faa7eb18ea30f932766efc Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Sun, 30 Jul 2023 11:05:03 +0200 Subject: [PATCH 2/8] Require version to be passed to "downloadVersion" Makes it possible to download different versions than the latest compatible one with retry functionality --- .../tachidesk/server/util/WebInterfaceManager.kt | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt index 685abce4f7..680ec7ac71 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt @@ -149,7 +149,7 @@ object WebInterfaceManager { */ val doDownload = { try { - downloadVersion() + downloadVersion(getLatestCompatibleVersion()) } catch (e: Exception) { false } || isLocalWebUIValid @@ -205,7 +205,7 @@ object WebInterfaceManager { } logger.info { "checkForUpdate(${serverConfig.webUIFlavor}, $localVersion): An update is available, starting download..." } - downloadVersion() + downloadVersion(getLatestCompatibleVersion()) } private fun getDownloadUrlFor(version: String): String { @@ -311,17 +311,15 @@ object WebInterfaceManager { throw Exception("No compatible webUI version found") } - fun downloadVersion(retryCount: Int = 0): Boolean { - val latestCompatibleVersion = getLatestCompatibleVersion() - - val webUIZip = "${WebUI.WEBUI.baseFileName}-$latestCompatibleVersion.zip" + fun downloadVersion(version: String, retryCount: Int = 0): Boolean { + val webUIZip = "${WebUI.WEBUI.baseFileName}-$version.zip" val webUIZipPath = "$tmpDir/$webUIZip" val webUIZipFile = File(webUIZipPath) - logger.info { "downloadVersion: Downloading WebUI (flavor= ${serverConfig.webUIFlavor}, version \"$latestCompatibleVersion\") zip from the Internet..." } + logger.info { "downloadVersion: Downloading WebUI (flavor= ${serverConfig.webUIFlavor}, version \"$version\") zip from the Internet..." } try { - val webUIZipURL = "${getDownloadUrlFor(latestCompatibleVersion)}/$webUIZip" + val webUIZipURL = "${getDownloadUrlFor(version)}/$webUIZip" downloadVersionZipFile(webUIZipURL, webUIZipFile) if (!isDownloadValid(webUIZip, webUIZipPath)) { @@ -332,7 +330,7 @@ object WebInterfaceManager { logger.error { "downloadVersion: Download failed${if (retry) ", retrying ${retryCount + 1}/3" else ""} - error: $e" } if (retry) { - return downloadVersion(retryCount + 1) + return downloadVersion(version, retryCount + 1) } return false From f75a88fbb791afe012e15fc56625a13216a5045d Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Sun, 30 Jul 2023 11:06:35 +0200 Subject: [PATCH 3/8] Fallback to downloading bundled webUI in case it's missing In case no download was possible and the fallback to the bundled version also failed due to it not existing, try to download the version of the bundled version as a last resort. --- .../server/util/WebInterfaceManager.kt | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt index 680ec7ac71..66f114f3a6 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt @@ -36,6 +36,8 @@ private val tmpDir = System.getProperty("java.io.tmpdir") private fun ByteArray.toHex(): String = joinToString(separator = "") { eachByte -> "%02x".format(eachByte) } +class BundledWebUIMissing : Exception("No bundled webUI version found") + enum class WebUIChannel { BUNDLED, // the default webUI version bundled with the server release STABLE, @@ -147,16 +149,16 @@ object WebInterfaceManager { * * In case the download failed but the local webUI is valid the download is considered a success to prevent the fallback logic */ - val doDownload = { + val doDownload: (version: String) -> Boolean = { version -> try { - downloadVersion(getLatestCompatibleVersion()) + downloadVersion(version) } catch (e: Exception) { false } || isLocalWebUIValid } // download the latest compatible version for the current selected webUI - val fallbackToDefaultWebUI = !doDownload() + val fallbackToDefaultWebUI = !doDownload(getLatestCompatibleVersion()) if (!fallbackToDefaultWebUI) { return } @@ -166,7 +168,7 @@ object WebInterfaceManager { serverConfig.webUIFlavor = DEFAULT_WEB_UI - val fallbackToBundledVersion = !doDownload() + val fallbackToBundledVersion = !doDownload(getLatestCompatibleVersion()) if (!fallbackToBundledVersion) { return } @@ -174,11 +176,21 @@ object WebInterfaceManager { logger.warn { "doInitialSetup: fallback to bundled default webUI \"$DEFAULT_WEB_UI\"" } - extractBundledWebUI() + try { + extractBundledWebUI() + return + } catch (e: BundledWebUIMissing) { + logger.warn(e) { "doInitialSetup: fallback to downloading the version of the bundled webUI" } + } + + val downloadFailed = !doDownload(BuildConfig.WEBUI_TAG) + if (downloadFailed) { + throw Exception("Unable to setup a webUI") + } } private fun extractBundledWebUI() { - val resourceWebUI: InputStream = BuildConfig::class.java.getResourceAsStream("/WebUI.zip") ?: throw Error("extractBundledWebUI: No bundled webUI version found") + val resourceWebUI: InputStream = BuildConfig::class.java.getResourceAsStream("/WebUI.zip") ?: throw BundledWebUIMissing() logger.info { "extractBundledWebUI: Using the bundled WebUI zip..." } From a4013305957a32b1133a846626b0bda1e799ccd7 Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Sun, 30 Jul 2023 11:20:08 +0200 Subject: [PATCH 4/8] Handle exception of "getLatestCompatibleVersion" --- .../tachidesk/server/util/WebInterfaceManager.kt | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt index 66f114f3a6..0baca6393c 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt @@ -149,16 +149,16 @@ object WebInterfaceManager { * * In case the download failed but the local webUI is valid the download is considered a success to prevent the fallback logic */ - val doDownload: (version: String) -> Boolean = { version -> + val doDownload: (getVersion: () -> String) -> Boolean = { getVersion -> try { - downloadVersion(version) + downloadVersion(getVersion()) } catch (e: Exception) { false } || isLocalWebUIValid } // download the latest compatible version for the current selected webUI - val fallbackToDefaultWebUI = !doDownload(getLatestCompatibleVersion()) + val fallbackToDefaultWebUI = !doDownload() { getLatestCompatibleVersion() } if (!fallbackToDefaultWebUI) { return } @@ -168,7 +168,7 @@ object WebInterfaceManager { serverConfig.webUIFlavor = DEFAULT_WEB_UI - val fallbackToBundledVersion = !doDownload(getLatestCompatibleVersion()) + val fallbackToBundledVersion = !doDownload() { getLatestCompatibleVersion() } if (!fallbackToBundledVersion) { return } @@ -183,7 +183,7 @@ object WebInterfaceManager { logger.warn(e) { "doInitialSetup: fallback to downloading the version of the bundled webUI" } } - val downloadFailed = !doDownload(BuildConfig.WEBUI_TAG) + val downloadFailed = !doDownload() { BuildConfig.WEBUI_TAG } if (downloadFailed) { throw Exception("Unable to setup a webUI") } @@ -217,7 +217,11 @@ object WebInterfaceManager { } logger.info { "checkForUpdate(${serverConfig.webUIFlavor}, $localVersion): An update is available, starting download..." } - downloadVersion(getLatestCompatibleVersion()) + try { + downloadVersion(getLatestCompatibleVersion()) + } catch (e: Exception) { + logger.warn(e) { "checkForUpdate: failed due to" } + } } private fun getDownloadUrlFor(version: String): String { From 1ad66d439380ab690fd07fdcaa1ef39fb6d45928 Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Sun, 30 Jul 2023 11:30:16 +0200 Subject: [PATCH 5/8] Move validation of download to actual download function --- .../server/util/WebInterfaceManager.kt | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt index 0baca6393c..0b1b28a845 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt @@ -330,20 +330,16 @@ object WebInterfaceManager { fun downloadVersion(version: String, retryCount: Int = 0): Boolean { val webUIZip = "${WebUI.WEBUI.baseFileName}-$version.zip" val webUIZipPath = "$tmpDir/$webUIZip" - val webUIZipFile = File(webUIZipPath) - logger.info { "downloadVersion: Downloading WebUI (flavor= ${serverConfig.webUIFlavor}, version \"$version\") zip from the Internet..." } + val log = KotlinLogging.logger("${logger.name} downloadVersion(version= $version, flavor= ${serverConfig.webUIFlavor})") + log.info { "Downloading WebUI zip from the Internet..." } try { val webUIZipURL = "${getDownloadUrlFor(version)}/$webUIZip" - downloadVersionZipFile(webUIZipURL, webUIZipFile) - - if (!isDownloadValid(webUIZip, webUIZipPath)) { - throw Exception("Download is invalid") - } + downloadVersionZipFile(webUIZipURL, webUIZipPath) } catch (e: Exception) { val retry = retryCount < 3 - logger.error { "downloadVersion: Download failed${if (retry) ", retrying ${retryCount + 1}/3" else ""} - error: $e" } + log.error { "failed${if (retry) ", retrying ${retryCount + 1}/3" else ""} - error: $e" } if (retry) { return downloadVersion(version, retryCount + 1) @@ -355,15 +351,17 @@ object WebInterfaceManager { File(applicationDirs.webUIRoot).deleteRecursively() // extract webUI zip - logger.info { "downloadVersion: Extracting WebUI zip..." } + log.info { "Extracting WebUI zip..." } extractDownload(webUIZipPath, applicationDirs.webUIRoot) - logger.info { "downloadVersion: Extracting WebUI zip Done." } + log.info { "Extracting WebUI zip Done." } return true } - private fun downloadVersionZipFile(url: String, zipFile: File) { + private fun downloadVersionZipFile(url: String, filePath: String) { + val zipFile = File(filePath) zipFile.delete() + val data = ByteArray(1024) zipFile.outputStream().use { webUIZipFileOut -> @@ -394,6 +392,10 @@ object WebInterfaceManager { logger.info { "downloadVersionZipFile: Downloading WebUI Done." } } } + + if (!isDownloadValid(zipFile.name, filePath)) { + throw Exception("Download is invalid") + } } private fun isDownloadValid(zipFileName: String, zipFilePath: String): Boolean { From 711f78795a24eda4383e12e61caac1f8e978c3cf Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Sun, 30 Jul 2023 11:45:28 +0200 Subject: [PATCH 6/8] Extract retry logic into function --- .../server/util/WebInterfaceManager.kt | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt index 0b1b28a845..913c7f481a 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt @@ -152,6 +152,7 @@ object WebInterfaceManager { val doDownload: (getVersion: () -> String) -> Boolean = { getVersion -> try { downloadVersion(getVersion()) + true } catch (e: Exception) { false } || isLocalWebUIValid @@ -275,6 +276,20 @@ object WebInterfaceManager { return digest.toHex() } + private fun executeWithRetry(log: KLogger, execute: () -> T, maxRetries: Int = 3, retryCount: Int = 0): T { + try { + return execute() + } catch (e: Exception) { + log.warn(e) { "(retry $retryCount/$maxRetries) failed due to" } + + if (retryCount < maxRetries) { + return executeWithRetry(log, execute, maxRetries, retryCount + 1) + } + + throw e + } + } + private fun fetchMD5SumFor(version: String): String { return try { val url = "${getDownloadUrlFor(version)}/md5sum" @@ -327,35 +342,21 @@ object WebInterfaceManager { throw Exception("No compatible webUI version found") } - fun downloadVersion(version: String, retryCount: Int = 0): Boolean { + fun downloadVersion(version: String) { val webUIZip = "${WebUI.WEBUI.baseFileName}-$version.zip" val webUIZipPath = "$tmpDir/$webUIZip" + val webUIZipURL = "${getDownloadUrlFor(version)}/$webUIZip" val log = KotlinLogging.logger("${logger.name} downloadVersion(version= $version, flavor= ${serverConfig.webUIFlavor})") log.info { "Downloading WebUI zip from the Internet..." } - try { - val webUIZipURL = "${getDownloadUrlFor(version)}/$webUIZip" - downloadVersionZipFile(webUIZipURL, webUIZipPath) - } catch (e: Exception) { - val retry = retryCount < 3 - log.error { "failed${if (retry) ", retrying ${retryCount + 1}/3" else ""} - error: $e" } - - if (retry) { - return downloadVersion(version, retryCount + 1) - } - - return false - } - + executeWithRetry(log, { downloadVersionZipFile(webUIZipURL, webUIZipPath) }) File(applicationDirs.webUIRoot).deleteRecursively() // extract webUI zip log.info { "Extracting WebUI zip..." } extractDownload(webUIZipPath, applicationDirs.webUIRoot) log.info { "Extracting WebUI zip Done." } - - return true } private fun downloadVersionZipFile(url: String, filePath: String) { From 66d384d7fc8fc98a7ec818364f792421388a3684 Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Sun, 30 Jul 2023 11:45:45 +0200 Subject: [PATCH 7/8] Retry every fetch up to 3 times --- .../server/util/WebInterfaceManager.kt | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt index 913c7f481a..45950f302c 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt @@ -11,6 +11,7 @@ import kotlinx.serialization.decodeFromString import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.jsonPrimitive +import mu.KLogger import mu.KotlinLogging import net.lingala.zip4j.ZipFile import org.json.JSONArray @@ -292,8 +293,10 @@ object WebInterfaceManager { private fun fetchMD5SumFor(version: String): String { return try { - val url = "${getDownloadUrlFor(version)}/md5sum" - URL(url).readText().trim() + executeWithRetry(KotlinLogging.logger("${logger.name} fetchMD5SumFor($version)"), { + val url = "${getDownloadUrlFor(version)}/md5sum" + URL(url).readText().trim() + }) } catch (e: Exception) { "" } @@ -305,8 +308,14 @@ object WebInterfaceManager { } private fun fetchPreviewVersion(): String { - val releaseInfoJson = URL(WebUI.WEBUI.latestReleaseInfoUrl).readText() - return Json.decodeFromString(releaseInfoJson)["tag_name"]?.jsonPrimitive?.content ?: throw Exception("Failed to get the preview version tag") + return executeWithRetry(KotlinLogging.logger("${logger.name} fetchPreviewVersion"), { + val releaseInfoJson = URL(WebUI.WEBUI.latestReleaseInfoUrl).readText() + Json.decodeFromString(releaseInfoJson)["tag_name"]?.jsonPrimitive?.content ?: throw Exception("Failed to get the preview version tag") + }) + } + + private fun fetchServerMappingFile(): JSONArray { + return executeWithRetry(KotlinLogging.logger("$logger fetchServerMappingFile"), { JSONArray(URL(WebUI.WEBUI.versionMappingUrl).readText()) }) } private fun getLatestCompatibleVersion(): String { @@ -316,7 +325,7 @@ object WebInterfaceManager { } val currentServerVersionNumber = extractVersion(BuildConfig.REVISION) - val webUIToServerVersionMappings = JSONArray(URL(WebUI.WEBUI.versionMappingUrl).readText()) + val webUIToServerVersionMappings = fetchServerMappingFile() logger.debug { "getLatestCompatibleVersion: webUIChannel= ${serverConfig.webUIChannel}, currentServerVersion= ${BuildConfig.REVISION}, mappingFile= $webUIToServerVersionMappings" } From 40fdc11a3338a96587ac22cb1441c132678b1eb2 Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Sun, 30 Jul 2023 12:14:19 +0200 Subject: [PATCH 8/8] Log full exception and change log level --- .../suwayomi/tachidesk/server/util/WebInterfaceManager.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt index 45950f302c..fcc42e3b7a 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt @@ -430,7 +430,7 @@ object WebInterfaceManager { val latestCompatibleVersion = getLatestCompatibleVersion() latestCompatibleVersion != currentVersion } catch (e: Exception) { - logger.debug { "isUpdateAvailable: check failed due to $e" } + logger.warn(e) { "isUpdateAvailable: check failed due to" } false } }