Skip to content

Commit c1d702a

Browse files
authored
Feature/improve automated backup (#597)
* Add option to disable cleanup of backups * Ensure the minimum TTL of backups to 1 day * Schedule the automated backup on a specific time of the day * Introduce scheduler that takes system hibernation time into account In case the system was hibernating/suspended scheduled task that should have been executed during that time would not get triggered and thus, miss an execution. To prevent this, this new scheduler periodically checks if the system was suspended and in case it was, triggers any task that missed its last execution * Use new scheduler
1 parent 0338ac3 commit c1d702a

File tree

8 files changed

+203
-50
lines changed

8 files changed

+203
-50
lines changed

gradle/libs.versions.toml

+6
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,12 @@ twelvemonkeys-imageio-webp = { module = "com.twelvemonkeys.imageio:imageio-webp"
128128
# Testing
129129
mockk = "io.mockk:mockk:1.13.2"
130130

131+
# cron scheduler
132+
cron4j = "it.sauronsoftware.cron4j:cron4j:2.2.5"
133+
134+
# cron-utils
135+
cronUtils = "com.cronutils:cron-utils:9.2.0"
136+
131137
[plugins]
132138
# Kotlin
133139
kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin"}

server/build.gradle.kts

+4
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,10 @@ dependencies {
7070
implementation("com.graphql-java:graphql-java-extended-scalars:20.0")
7171

7272
testImplementation(libs.mockk)
73+
74+
implementation(libs.cron4j)
75+
76+
implementation(libs.cronUtils)
7377
}
7478

7579
application {

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt

+28-28
Original file line numberDiff line numberDiff line change
@@ -36,54 +36,50 @@ import suwayomi.tachidesk.manga.model.table.SourceTable
3636
import suwayomi.tachidesk.manga.model.table.toDataClass
3737
import suwayomi.tachidesk.server.ApplicationDirs
3838
import suwayomi.tachidesk.server.serverConfig
39+
import suwayomi.tachidesk.util.HAScheduler
3940
import java.io.ByteArrayOutputStream
4041
import java.io.File
4142
import java.io.InputStream
4243
import java.text.SimpleDateFormat
4344
import java.util.Date
44-
import java.util.Timer
45-
import java.util.TimerTask
4645
import java.util.concurrent.TimeUnit
4746
import java.util.prefs.Preferences
4847
import kotlin.time.Duration.Companion.days
4948

5049
object ProtoBackupExport : ProtoBackupBase() {
5150
private val logger = KotlinLogging.logger { }
5251
private val applicationDirs by DI.global.instance<ApplicationDirs>()
52+
private var backupSchedulerJobId: String = ""
5353
private const val lastAutomatedBackupKey = "lastAutomatedBackupKey"
5454
private val preferences = Preferences.userNodeForPackage(ProtoBackupExport::class.java)
5555

56-
private val backupTimer = Timer()
57-
private var currentAutomatedBackupTask: TimerTask? = null
58-
5956
fun scheduleAutomatedBackupTask() {
57+
HAScheduler.deschedule(backupSchedulerJobId)
58+
6059
if (!serverConfig.automatedBackups) {
61-
currentAutomatedBackupTask?.cancel()
6260
return
6361
}
6462

65-
val minInterval = 1.days
66-
val interval = serverConfig.backupInterval.days
67-
val backupInterval = interval.coerceAtLeast(minInterval).inWholeMilliseconds
68-
69-
val lastAutomatedBackup = preferences.getLong(lastAutomatedBackupKey, 0)
70-
val initialDelay =
71-
backupInterval - (System.currentTimeMillis() - lastAutomatedBackup) % backupInterval
63+
val task = {
64+
cleanupAutomatedBackups()
65+
createAutomatedBackup()
66+
preferences.putLong(lastAutomatedBackupKey, System.currentTimeMillis())
67+
}
7268

73-
currentAutomatedBackupTask?.cancel()
74-
currentAutomatedBackupTask = object : TimerTask() {
75-
override fun run() {
76-
cleanupAutomatedBackups()
77-
createAutomatedBackup()
78-
preferences.putLong(lastAutomatedBackupKey, System.currentTimeMillis())
79-
}
69+
val (hour, minute) = serverConfig.backupTime.split(":").map { it.toInt() }
70+
val backupHour = hour.coerceAtLeast(0).coerceAtMost(23)
71+
val backupMinute = minute.coerceAtLeast(0).coerceAtMost(59)
72+
val backupInterval = serverConfig.backupInterval.days.coerceAtLeast(1.days)
73+
74+
// trigger last backup in case the server wasn't running on the scheduled time
75+
val lastAutomatedBackup = preferences.getLong(lastAutomatedBackupKey, System.currentTimeMillis())
76+
val wasPreviousBackupTriggered =
77+
(System.currentTimeMillis() - lastAutomatedBackup) < backupInterval.inWholeMilliseconds
78+
if (!wasPreviousBackupTriggered) {
79+
task()
8080
}
8181

82-
backupTimer.scheduleAtFixedRate(
83-
currentAutomatedBackupTask,
84-
initialDelay,
85-
backupInterval
86-
)
82+
HAScheduler.schedule(task, "$backupMinute $backupHour */${backupInterval.inWholeDays} * *", "backup")
8783
}
8884

8985
private fun createAutomatedBackup() {
@@ -105,11 +101,15 @@ object ProtoBackupExport : ProtoBackupBase() {
105101

106102
backupFile.outputStream().use { output -> input.copyTo(output) }
107103
}
108-
109104
}
110105

111106
private fun cleanupAutomatedBackups() {
112-
logger.debug { "Cleanup automated backups" }
107+
logger.debug { "Cleanup automated backups (ttl= ${serverConfig.backupTTL})" }
108+
109+
val isCleanupDisabled = serverConfig.backupTTL == 0
110+
if (isCleanupDisabled) {
111+
return
112+
}
113113

114114
val automatedBackupDir = File(applicationDirs.automatedBackupRoot)
115115
if (!automatedBackupDir.isDirectory) {
@@ -132,7 +132,7 @@ object ProtoBackupExport : ProtoBackupBase() {
132132

133133
val lastAccessTime = file.lastModified()
134134
val isTTLReached =
135-
System.currentTimeMillis() - lastAccessTime >= serverConfig.backupTTL.days.inWholeMilliseconds
135+
System.currentTimeMillis() - lastAccessTime >= serverConfig.backupTTL.days.coerceAtLeast(1.days).inWholeMilliseconds
136136
if (isTTLReached) {
137137
file.delete()
138138
}

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt

+30-21
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package suwayomi.tachidesk.manga.impl.update
22

33
import eu.kanade.tachiyomi.source.model.UpdateStrategy
4+
import it.sauronsoftware.cron4j.Task
5+
import it.sauronsoftware.cron4j.TaskExecutionContext
46
import kotlinx.coroutines.CancellationException
57
import kotlinx.coroutines.CoroutineScope
68
import kotlinx.coroutines.Dispatchers
@@ -24,11 +26,13 @@ import org.kodein.di.instance
2426
import suwayomi.tachidesk.manga.impl.Category
2527
import suwayomi.tachidesk.manga.impl.CategoryManga
2628
import suwayomi.tachidesk.manga.impl.Chapter
29+
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupExport
2730
import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
2831
import suwayomi.tachidesk.manga.model.dataclass.IncludeInUpdate
2932
import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
3033
import suwayomi.tachidesk.manga.model.table.MangaStatus
3134
import suwayomi.tachidesk.server.serverConfig
35+
import suwayomi.tachidesk.util.HAScheduler
3236
import java.util.Date
3337
import java.util.Timer
3438
import java.util.TimerTask
@@ -51,41 +55,46 @@ class Updater : IUpdater {
5155
private val lastAutomatedUpdateKey = "lastAutomatedUpdateKey"
5256
private val preferences = Preferences.userNodeForPackage(Updater::class.java)
5357

54-
private val updateTimer = Timer()
55-
private var currentUpdateTask: TimerTask? = null
58+
private var currentUpdateTaskId = ""
5659

5760
init {
5861
scheduleUpdateTask()
5962
}
6063

64+
65+
private fun autoUpdateTask() {
66+
val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0)
67+
preferences.putLong(lastAutomatedUpdateKey, System.currentTimeMillis())
68+
69+
if (status.value.running) {
70+
logger.debug { "Global update is already in progress" }
71+
return
72+
}
73+
74+
logger.info { "Trigger global update (interval= ${serverConfig.globalUpdateInterval}h, lastAutomatedUpdate= ${Date(lastAutomatedUpdate)})" }
75+
addCategoriesToUpdateQueue(Category.getCategoryList(), true)
76+
}
77+
6178
private fun scheduleUpdateTask() {
79+
HAScheduler.deschedule(currentUpdateTaskId)
80+
6281
if (!serverConfig.automaticallyTriggerGlobalUpdate) {
6382
return
6483
}
6584

6685
val minInterval = 6.hours
6786
val interval = serverConfig.globalUpdateInterval.hours
68-
val updateInterval = interval.coerceAtLeast(minInterval).inWholeMilliseconds
69-
70-
val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0)
71-
val initialDelay = updateInterval - (System.currentTimeMillis() - lastAutomatedUpdate) % updateInterval
72-
73-
currentUpdateTask?.cancel()
74-
currentUpdateTask = object : TimerTask() {
75-
override fun run() {
76-
preferences.putLong(lastAutomatedUpdateKey, System.currentTimeMillis())
77-
78-
if (status.value.running) {
79-
logger.debug { "Global update is already in progress, do not trigger global update" }
80-
return
81-
}
82-
83-
logger.info { "Trigger global update (interval= ${serverConfig.globalUpdateInterval}h, lastAutomatedUpdate= ${Date(lastAutomatedUpdate)})" }
84-
addCategoriesToUpdateQueue(Category.getCategoryList(), true)
85-
}
87+
val updateInterval = interval.coerceAtLeast(minInterval)
88+
val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, System.currentTimeMillis())
89+
90+
// trigger update in case the server wasn't running on the scheduled time
91+
val wasPreviousUpdateTriggered =
92+
(System.currentTimeMillis() - lastAutomatedUpdate) < updateInterval.inWholeMilliseconds
93+
if (!wasPreviousUpdateTriggered) {
94+
autoUpdateTask()
8695
}
8796

88-
updateTimer.scheduleAtFixedRate(currentUpdateTask, initialDelay, updateInterval)
97+
HAScheduler.schedule(::autoUpdateTask, "* */${updateInterval.inWholeHours} * * *", "global-update")
8998
}
9099

91100
private fun getOrCreateUpdateChannelFor(source: String): Channel<UpdateJob> {

server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt

+1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ class ServerConfig(getConfig: () -> Config, moduleName: String = MODULE_NAME) :
5252

5353
// backup
5454
var backupPath: String by overridableConfig
55+
var backupTime: String by overridableConfig
5556
var backupInterval: Int by overridableConfig
5657
var automatedBackups: Boolean by overridableConfig
5758
var backupTTL: Int by overridableConfig
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
2+
package suwayomi.tachidesk.util
3+
4+
import com.cronutils.model.CronType.CRON4J
5+
import com.cronutils.model.definition.CronDefinitionBuilder
6+
import com.cronutils.model.time.ExecutionTime
7+
import com.cronutils.parser.CronParser
8+
import it.sauronsoftware.cron4j.Scheduler
9+
import it.sauronsoftware.cron4j.Task
10+
import it.sauronsoftware.cron4j.TaskExecutionContext
11+
import mu.KotlinLogging
12+
import java.time.ZonedDateTime
13+
import java.util.PriorityQueue
14+
import java.util.Timer
15+
import java.util.TimerTask
16+
import kotlin.time.Duration
17+
import kotlin.time.Duration.Companion.minutes
18+
import kotlin.time.Duration.Companion.seconds
19+
20+
val cronParser = CronParser(CronDefinitionBuilder.instanceDefinitionFor(CRON4J))
21+
22+
class HATask(val id: String, val cronExpr: String, val execute: () -> Unit, val name: String?) : Comparable<HATask> {
23+
private val executionTime = ExecutionTime.forCron(cronParser.parse(cronExpr))
24+
25+
fun getLastExecutionTime(): Long {
26+
return executionTime.lastExecution(ZonedDateTime.now()).get().toEpochSecond().seconds.inWholeMilliseconds
27+
}
28+
29+
fun getNextExecutionTime(): Long {
30+
return executionTime.nextExecution(ZonedDateTime.now()).get().toEpochSecond().seconds.inWholeMilliseconds
31+
}
32+
33+
fun getTimeToNextExecution(): Long {
34+
return executionTime.timeToNextExecution(ZonedDateTime.now()).get().toMillis()
35+
}
36+
37+
override fun compareTo(other: HATask): Int {
38+
return getTimeToNextExecution().compareTo(other.getTimeToNextExecution())
39+
}
40+
}
41+
42+
/**
43+
* The "HAScheduler" ("HibernateAwareScheduler") is a scheduler that recognizes when the system was hibernating/suspended
44+
* and triggers tasks that have missed their execution points.
45+
*/
46+
object HAScheduler {
47+
private val logger = KotlinLogging.logger { }
48+
49+
private val scheduledTasks = PriorityQueue<HATask>()
50+
private val scheduler = Scheduler()
51+
52+
private val HIBERNATION_THRESHOLD = 10.seconds.inWholeMilliseconds
53+
private const val TASK_THRESHOLD = 0.1
54+
55+
init {
56+
scheduleHibernateCheckerTask(1.minutes)
57+
}
58+
59+
private fun scheduleHibernateCheckerTask(interval: Duration) {
60+
val timer = Timer()
61+
timer.scheduleAtFixedRate(
62+
object : TimerTask() {
63+
var lastExecutionTime = System.currentTimeMillis()
64+
65+
override fun run() {
66+
val currentTime = System.currentTimeMillis()
67+
val elapsedTime = currentTime - lastExecutionTime
68+
lastExecutionTime = currentTime
69+
70+
val systemWasInHibernation = elapsedTime > interval.inWholeMilliseconds + HIBERNATION_THRESHOLD
71+
if (systemWasInHibernation) {
72+
logger.debug { "System hibernation detected, task was delayed by ${elapsedTime - interval.inWholeMilliseconds}ms" }
73+
scheduledTasks.forEach {
74+
val missedExecution = currentTime - it.getLastExecutionTime() - elapsedTime < 0
75+
val taskInterval = it.getNextExecutionTime() - it.getLastExecutionTime()
76+
// in case the next task execution doesn't take long the missed execution can be ignored to prevent a double execution
77+
val taskThresholdMet = taskInterval * TASK_THRESHOLD > it.getTimeToNextExecution()
78+
79+
val triggerTask = missedExecution && taskThresholdMet
80+
if (triggerTask) {
81+
logger.debug { "Task \"${it.name ?: it.id}\" missed its execution, executing now..." }
82+
reschedule(it.id, it.cronExpr)
83+
it.execute()
84+
}
85+
86+
// queue is ordered by next execution time, thus, loop can be exited early
87+
if (!missedExecution) {
88+
return@forEach
89+
}
90+
}
91+
}
92+
}
93+
},
94+
interval.inWholeMilliseconds,
95+
interval.inWholeMilliseconds
96+
)
97+
}
98+
99+
fun schedule(execute: () -> Unit, cronExpr: String, name: String?): String {
100+
if (!scheduler.isStarted) {
101+
scheduler.start()
102+
}
103+
104+
val taskId = scheduler.schedule(
105+
cronExpr,
106+
object : Task() {
107+
override fun execute(context: TaskExecutionContext?) {
108+
execute()
109+
}
110+
}
111+
)
112+
113+
scheduledTasks.add(HATask(taskId, cronExpr, execute, name))
114+
115+
return taskId
116+
}
117+
118+
fun deschedule(taskId: String) {
119+
scheduler.deschedule(taskId)
120+
scheduledTasks.removeIf { it.id == taskId }
121+
}
122+
123+
fun reschedule(taskId: String, cronExpr: String) {
124+
val task = scheduledTasks.find { it.id == taskId } ?: return
125+
126+
scheduledTasks.remove(task)
127+
scheduledTasks.add(HATask(taskId, cronExpr, task.execute, task.name))
128+
129+
scheduler.reschedule(taskId, cronExpr)
130+
}
131+
}

server/src/main/resources/server-reference.conf

+2-1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ server.systemTrayEnabled = true
3737

3838
# backup
3939
server.backupPath = ""
40+
server.backupTime = "00:00" # range: hour: 0-23, minute: 0-59 - default: "00:00" - time of day at which the automated backup should be triggered
4041
server.backupInterval = 1 # time in days - range: 1 <= n < ∞ - default: 1 day - interval in which the server will automatically create a backup
4142
server.automatedBackups = true
42-
server.backupTTL = 14 # time in days - range: 1 <= n < ∞ - default: 14 days - how long backup files will be kept before they will get deleted
43+
server.backupTTL = 14 # time in days - 0 to disable it - range: 1 <= n < ∞ - default: 14 days - how long backup files will be kept before they will get deleted

server/src/test/resources/server-reference.conf

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ server.electronPath = ""
3030

3131
# backup
3232
server.backupPath = ""
33+
server.backupTime = "00:00"
3334
server.backupInterval = 1
3435
server.automatedBackups = true
3536
server.backupTTL = 14

0 commit comments

Comments
 (0)