From d36c7719e6f05a76de037bd4a7ed1b68af4a2a72 Mon Sep 17 00:00:00 2001 From: MrDwarf7 <129040985+MrDwarf7@users.noreply.github.com> Date: Tue, 7 Jan 2025 13:54:55 +1100 Subject: [PATCH 1/5] chore(Cargo): add dependencies to support graceful shutdown --- Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 85f7f59..f524a3c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,8 @@ tera = "1.20.0" test-context = "0.3.0" thiserror = "2.0.6" tokio = { version = "1.42.0", features = ["full"] } +tower = { version = "0.5.1", features = ["full"] } +tower-http = { version = "0.6.2", features = ["full"] } tracing = { version = "0.1.41", features = ["attributes"] } tracing-appender = "0.2.3" tracing-bunyan-formatter = "0.3.10" @@ -76,4 +78,4 @@ uuid = { version = "1.11.0", features = ["v4", "serde"] } tokio-tungstenite = "0.24.0" garde = { version = "0.20.0", features = ["full"] } regex = "1.11.1" -wiremock = "0.6.2" \ No newline at end of file +wiremock = "0.6.2" From 38c6008366adfb8e321aa8d81e30cea8b20a9e26 Mon Sep 17 00:00:00 2001 From: MrDwarf7 <129040985+MrDwarf7@users.noreply.github.com> Date: Tue, 7 Jan 2025 13:56:59 +1100 Subject: [PATCH 2/5] feat(settings): add graceful shutdown to settings files feat(configure/server): add graceful shutdown to server struct --- settings/base.toml | 1 + settings/dev.toml | 1 + settings/prod.toml | 1 + settings/test.toml | 1 + src/configure/server.rs | 2 ++ 5 files changed, 6 insertions(+) diff --git a/settings/base.toml b/settings/base.toml index b419aeb..f4296df 100644 --- a/settings/base.toml +++ b/settings/base.toml @@ -1,6 +1,7 @@ [server] addr = "0.0.0.0" port = 8_080 +grace_shutdown_secs = 10 [db] host = "127.0.0.1" diff --git a/settings/dev.toml b/settings/dev.toml index 8519e7b..f120748 100644 --- a/settings/dev.toml +++ b/settings/dev.toml @@ -3,6 +3,7 @@ profile = "dev" [server] addr = "127.0.0.1" port = 8_080 +grace_shutdown = 10 [db] host = "127.0.0.1" diff --git a/settings/prod.toml b/settings/prod.toml index fe8fc86..169897f 100644 --- a/settings/prod.toml +++ b/settings/prod.toml @@ -3,6 +3,7 @@ profile = "prod" [server] addr = "0.0.0.0" port = 8_080 +grace_shutdown = 30 [db] host = "127.0.0.1" diff --git a/settings/test.toml b/settings/test.toml index 6f8af9b..61de3fb 100644 --- a/settings/test.toml +++ b/settings/test.toml @@ -3,6 +3,7 @@ profile = "test" [server] addr = "127.0.0.1" port = 0 +grace_shutdown = 1 [db] host = "127.0.0.1" diff --git a/src/configure/server.rs b/src/configure/server.rs index 6eaf085..d2223da 100644 --- a/src/configure/server.rs +++ b/src/configure/server.rs @@ -6,6 +6,7 @@ use serde::Deserialize; pub struct ServerConfig { pub addr: String, pub port: u16, + pub grace_shutdown_secs: i64, } impl ServerConfig { @@ -31,6 +32,7 @@ pub mod tests { let config = ServerConfig { addr: "127.0.0.1".to_string(), port: 1024, + grace_shutdown_secs: 10, }; assert_eq!(config.get_http_addr(), "http://127.0.0.1:1024"); } From fc8f9a0b9d2ded47e8fb81e3c06602bad255f76a Mon Sep 17 00:00:00 2001 From: MrDwarf7 <129040985+MrDwarf7@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:01:20 +1100 Subject: [PATCH 3/5] fix(util/task): allow server to correctly exit now that graceful shutdown is impl --- src/util/task.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/util/task.rs b/src/util/task.rs index 7293d0c..f54f366 100644 --- a/src/util/task.rs +++ b/src/util/task.rs @@ -17,18 +17,20 @@ pub async fn join_all(tasks: Vec) -> AppResult { tokio::spawn(async { if let Err(e) = task.await { if let Some(sender) = sender { - sender - .send(e) - .await - .unwrap_or_else(|_| unreachable!("This channel never closed.")); + let _ = sender.send(e).await; } else { error!("A task failed: {e}."); } } }); } + + // Explicitly drop the sender to close the channel. + drop(sender); + + // Return Ok(()) if all futures are completed without error. match receiver.recv().await { Some(err) => Err(err), - None => unreachable!("This channel never closed."), + None => Ok(()), } } From f9e17c9f10bf9dd60ba668da2ad1b63944c571d0 Mon Sep 17 00:00:00 2001 From: MrDwarf7 <129040985+MrDwarf7@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:00:11 +1100 Subject: [PATCH 4/5] feat(server::run): add graceful shutdown implementation --- src/server/mod.rs | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/src/server/mod.rs b/src/server/mod.rs index 86acefe..339a329 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -2,6 +2,9 @@ use self::state::AppState; use crate::configure::AppConfig; use crate::error::AppResult; use crate::router::create_router_app; +use tower_http::timeout::TimeoutLayer; +use tower_http::trace::TraceLayer; +use tracing::info; pub mod state; pub mod worker; @@ -20,8 +23,46 @@ impl AppServer { } pub async fn run(self) -> AppResult<()> { - let router = create_router_app(self.state); - axum::serve(self.tcp, router).await?; + let shutdown = self.grace_shutdown_time(); + let router = create_router_app(self.state) + .layer(TraceLayer::new_for_http()) // Visibility of the request and response, change as needed. + .layer(TimeoutLayer::new(shutdown)); // Graceful shutdown for hosting services requries N time to complete the request before shutting down. + + axum::serve(self.tcp, router) + .with_graceful_shutdown(shutdown_signal()) + .await?; Ok(()) } + + fn grace_shutdown_time(&self) -> std::time::Duration { + std::time::Duration::from_secs(self.state.config.server.grace_shutdown_secs as u64) + } +} + +async fn shutdown_signal() { + let ctrl_c = async { + tokio::signal::ctrl_c() + .await + .expect("Failed to install CTRL+C signal handler"); + }; + + #[cfg(unix)] + let terminate = async { + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("Failed to install SIGTERM signal handler") + .recv() + .await; + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => { + info!("Received Ctrl-C signal. Shutting down..."); + }, + _ = terminate => { + info!("Received SIGTERM signal. Shutting down..."); + }, + } } From 17d19648f047db5fa057abf0eeaff3ce13c98f48 Mon Sep 17 00:00:00 2001 From: MrDwarf7 <129040985+MrDwarf7@users.noreply.github.com> Date: Sun, 19 Jan 2025 12:35:38 +1100 Subject: [PATCH 5/5] fix(settings/*.toml): Update grace_shutdown to grace_shutdown_secs Reason: Configuration option was previously named "grace_shutdown" this was updated to "grace_shutdown_secs" for clarity. Originally all settings files were not updated, this has now been corrected to align with the configuration struct in `src/configure/server.rs` --- settings/dev.toml | 2 +- settings/prod.toml | 2 +- settings/test.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/settings/dev.toml b/settings/dev.toml index f120748..a219e1c 100644 --- a/settings/dev.toml +++ b/settings/dev.toml @@ -3,7 +3,7 @@ profile = "dev" [server] addr = "127.0.0.1" port = 8_080 -grace_shutdown = 10 +grace_shutdown_secs = 10 [db] host = "127.0.0.1" diff --git a/settings/prod.toml b/settings/prod.toml index 169897f..0a13b21 100644 --- a/settings/prod.toml +++ b/settings/prod.toml @@ -3,7 +3,7 @@ profile = "prod" [server] addr = "0.0.0.0" port = 8_080 -grace_shutdown = 30 +grace_shutdown_secs = 30 [db] host = "127.0.0.1" diff --git a/settings/test.toml b/settings/test.toml index 61de3fb..05ea43d 100644 --- a/settings/test.toml +++ b/settings/test.toml @@ -3,7 +3,7 @@ profile = "test" [server] addr = "127.0.0.1" port = 0 -grace_shutdown = 1 +grace_shutdown_secs = 1 [db] host = "127.0.0.1"