From f30cd49c3e9b542f6440cda00efc144878b122b3 Mon Sep 17 00:00:00 2001 From: Miguel Medeiros Date: Sat, 8 Feb 2025 23:32:17 -0300 Subject: [PATCH 1/7] feat: add examples for DHT server and message formatting --- Cargo.toml | 1 + examples/README.md | 44 +++++++++++-- examples/node.rs | 157 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 examples/node.rs diff --git a/Cargo.toml b/Cargo.toml index b9279875..b0716872 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ ed25519-dalek = "2.1.1" tracing = "0.1" lru = { version = "0.12.5", default-features = false } dyn-clone = "1.0.18" +colored = "2.0.0" document-features = "0.2.10" diff --git a/examples/README.md b/examples/README.md index e9f44e94..6e2baf07 100644 --- a/examples/README.md +++ b/examples/README.md @@ -36,14 +36,50 @@ cargo run --example put_mutable <64 bytes hex secret_key> cargo run --example get_mutable <40 bytes hex target from put_mutable> ``` -## Custom Server +## Request Filter + +Example showing how to implement a custom request filter for the DHT server: + +```sh +cargo run --example request_filter +``` + +## Cache Bootstrap + +Example demonstrating how to cache and reuse bootstrapping nodes: ```sh -cargo run --example custom_server -```` +cargo run --example cache_bootstrap +``` + +## Count IPs Close to Key -## Measure the Dht size +Count all IP addresses around a random target ID and analyze hit rates: + +```sh +cargo run --example count_ips_close_to_key +``` + +## Mark Recapture DHT + +Estimate DHT size using Mark-Recapture method: + +```sh +cargo run --example mark_recapture_dht +``` + +## Measure DHT + +Measure the size of the DHT network: ```sh cargo run --example measure_dht ``` + +## Bootstrap + +Basic example of bootstrapping a DHT node: + +```sh +cargo run --example bootstrap +``` diff --git a/examples/node.rs b/examples/node.rs new file mode 100644 index 00000000..034a568c --- /dev/null +++ b/examples/node.rs @@ -0,0 +1,157 @@ +use mainline::Dht; +use std::{thread, time::Duration}; +use tracing::{Level, info, debug, trace}; +use tracing_subscriber; +use colored::*; + +/// Format a DHT message for better readability +fn format_dht_message(msg: &str) -> String { + // Extract message type and content + if msg.contains("Sending") { + format_outgoing_message(msg) + } else if msg.contains("Received") { + format_incoming_message(msg) + } else { + msg.to_string() + } +} + +/// Format outgoing messages +fn format_outgoing_message(msg: &str) -> String { + let arrow = "→".bright_blue(); + format_message(msg, &arrow) +} + +/// Format incoming messages +fn format_incoming_message(msg: &str) -> String { + let arrow = "←".bright_green(); + format_message(msg, &arrow) +} + +/// Common message formatting logic +fn format_message(msg: &str, arrow: &str) -> String { + // Extract message type (find_node, get_peers, etc) + let msg_type = if msg.contains("find_node") { + "FIND_NODE".yellow() + } else if msg.contains("get_peers") { + "GET_PEERS".purple() + } else if msg.contains("announce_peer") { + "ANNOUNCE".cyan() + } else if msg.contains("ping") { + "PING".bright_black() + } else { + "OTHER".white() + }; + + // Format the message in a structured way + format!( + "\n{} {} | {}\n{}", + arrow, + msg_type, + "DHT Message".bright_black(), + format_message_details(msg) + ) +} + +/// Format message details in a structured way +fn format_message_details(msg: &str) -> String { + let mut details = String::new(); + + // Extract and format common fields + if let Some(id) = extract_field(msg, "id=") { + details.push_str(&format!(" ID: {}\n", id.bright_yellow())); + } + if let Some(addr) = extract_field(msg, "address=") { + details.push_str(&format!(" Address: {}\n", addr.bright_blue())); + } + if let Some(nodes) = extract_field(msg, "nodes=") { + details.push_str(&format!(" Nodes: {}\n", format_nodes(nodes))); + } + if let Some(values) = extract_field(msg, "values=") { + details.push_str(&format!(" Values: {}\n", values.bright_magenta())); + } + if let Some(token) = extract_field(msg, "token=") { + details.push_str(&format!(" Token: {}\n", token.bright_yellow())); + } + + details +} + +/// Extract a field from the message +fn extract_field<'a>(msg: &'a str, field: &str) -> Option<&'a str> { + msg.split(field).nth(1).map(|s| s.split_whitespace().next().unwrap_or("")) +} + +/// Format nodes list for better readability +fn format_nodes(nodes: &str) -> String { + nodes + .split(", ") + .take(3) + .map(|n| n.bright_cyan().to_string()) + .collect::>() + .join(", ") + + if nodes.split(", ").count() > 3 { " ..." } else { "" } +} + +fn main() { + // Configure logging to see ALL details + tracing_subscriber::fmt() + .with_max_level(Level::TRACE) + .with_thread_ids(true) + .with_target(true) + .with_file(true) + .with_ansi(true) + .with_line_number(true) + .with_thread_names(true) + .init(); + + // Configure and start the DHT node in server mode + let dht = Dht::builder() + .server_mode() + .build() + .expect("Failed to create DHT server"); + + info!("DHT server node is running! Press Ctrl+C to stop."); + + // Wait for bootstrap to complete + info!("Waiting for bootstrap..."); + dht.bootstrapped(); + info!("Bootstrap complete!"); + + // Keep the program running and show periodic information + loop { + thread::sleep(Duration::from_secs(30)); + let info = dht.info(); + + // Basic node information + info!("=== DHT Node Status ==="); + info!("Node ID: {}", info.id().to_string().yellow()); + info!("Local address: {}", info.local_addr().to_string().blue()); + if let Some(addr) = info.public_address() { + info!("Public address: {}", addr.to_string().green()); + } + info!("Firewalled: {}", if info.firewalled() { "Yes".red() } else { "No".green() }); + info!("Server mode: {}", if info.server_mode() { "Yes".green() } else { "No".yellow() }); + + // Network statistics + let (size_estimate, std_dev) = info.dht_size_estimate(); + info!("=== Network Statistics ==="); + info!( + "Estimated nodes in network: {} (±{:.1}%)", + size_estimate.to_string().cyan(), + (std_dev * 100.0).to_string().yellow() + ); + + debug!("Raw DHT Info: {:?}", info); + + // Add explanation of message types + trace!("Message Types you might see in logs:"); + trace!("- find_node: Looking for nodes close to an ID"); + trace!("- get_peers: Looking for peers for an infohash"); + trace!("- announce_peer: Announcing that it's a peer for an infohash"); + trace!("- ping: Checking if a node is alive"); + trace!("- NoValues: Response indicating no requested values were found"); + + info!(""); // Blank line to separate updates + } +} \ No newline at end of file From 7b24451767cf53312be470ee3f87f849b9c0bd53 Mon Sep 17 00:00:00 2001 From: Miguel Medeiros Date: Sat, 8 Feb 2025 23:59:09 -0300 Subject: [PATCH 2/7] feat: enhance DHT message logging with custom formatting --- examples/node.rs | 91 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 30 deletions(-) diff --git a/examples/node.rs b/examples/node.rs index 034a568c..3df9a22a 100644 --- a/examples/node.rs +++ b/examples/node.rs @@ -1,9 +1,26 @@ use mainline::Dht; use std::{thread, time::Duration}; use tracing::{Level, info, debug, trace}; -use tracing_subscriber; +use tracing_subscriber::{self, fmt::format::FmtSpan}; use colored::*; +/// Custom logger that formats DHT messages +struct DhtLogger; + +impl DhtLogger { + fn info(msg: &str) { + info!("{}", format_dht_message(msg)); + } + + fn debug(msg: &str) { + debug!("{}", format_dht_message(msg)); + } + + fn trace(msg: &str) { + trace!("{}", format_dht_message(msg)); + } +} + /// Format a DHT message for better readability fn format_dht_message(msg: &str) -> String { // Extract message type and content @@ -18,18 +35,16 @@ fn format_dht_message(msg: &str) -> String { /// Format outgoing messages fn format_outgoing_message(msg: &str) -> String { - let arrow = "→".bright_blue(); - format_message(msg, &arrow) + format_message(msg, "OUT".bright_blue()) } /// Format incoming messages fn format_incoming_message(msg: &str) -> String { - let arrow = "←".bright_green(); - format_message(msg, &arrow) + format_message(msg, "IN".bright_green()) } /// Common message formatting logic -fn format_message(msg: &str, arrow: &str) -> String { +fn format_message(msg: &str, direction: ColoredString) -> String { // Extract message type (find_node, get_peers, etc) let msg_type = if msg.contains("find_node") { "FIND_NODE".yellow() @@ -45,8 +60,8 @@ fn format_message(msg: &str, arrow: &str) -> String { // Format the message in a structured way format!( - "\n{} {} | {}\n{}", - arrow, + "[{}] {} | {}\n{}", + direction, msg_type, "DHT Message".bright_black(), format_message_details(msg) @@ -103,6 +118,16 @@ fn main() { .with_ansi(true) .with_line_number(true) .with_thread_names(true) + .with_span_events(FmtSpan::FULL) + .event_format( + tracing_subscriber::fmt::format() + .with_thread_ids(true) + .with_thread_names(true) + .with_file(true) + .with_line_number(true) + .with_target(true) + .compact() // This will make the output more compact + ) .init(); // Configure and start the DHT node in server mode @@ -111,12 +136,12 @@ fn main() { .build() .expect("Failed to create DHT server"); - info!("DHT server node is running! Press Ctrl+C to stop."); + DhtLogger::info("DHT server node is running! Press Ctrl+C to stop."); // Wait for bootstrap to complete - info!("Waiting for bootstrap..."); + DhtLogger::info("Waiting for bootstrap..."); dht.bootstrapped(); - info!("Bootstrap complete!"); + DhtLogger::info("Bootstrap complete!"); // Keep the program running and show periodic information loop { @@ -124,34 +149,40 @@ fn main() { let info = dht.info(); // Basic node information - info!("=== DHT Node Status ==="); - info!("Node ID: {}", info.id().to_string().yellow()); - info!("Local address: {}", info.local_addr().to_string().blue()); + DhtLogger::info("=== DHT Node Status ==="); + DhtLogger::info(&format!("Node ID: {}", info.id().to_string().yellow())); + DhtLogger::info(&format!("Local address: {}", info.local_addr().to_string().blue())); if let Some(addr) = info.public_address() { - info!("Public address: {}", addr.to_string().green()); + DhtLogger::info(&format!("Public address: {}", addr.to_string().green())); } - info!("Firewalled: {}", if info.firewalled() { "Yes".red() } else { "No".green() }); - info!("Server mode: {}", if info.server_mode() { "Yes".green() } else { "No".yellow() }); + DhtLogger::info(&format!( + "Firewalled: {}", + if info.firewalled() { "Yes".red() } else { "No".green() } + )); + DhtLogger::info(&format!( + "Server mode: {}", + if info.server_mode() { "Yes".green() } else { "No".yellow() } + )); // Network statistics let (size_estimate, std_dev) = info.dht_size_estimate(); - info!("=== Network Statistics ==="); - info!( - "Estimated nodes in network: {} (±{:.1}%)", + DhtLogger::info("=== Network Statistics ==="); + DhtLogger::info(&format!( + "Estimated nodes in network: {} (±{}%)", size_estimate.to_string().cyan(), - (std_dev * 100.0).to_string().yellow() - ); + format!("{:.1}", std_dev * 100.0).yellow() + )); - debug!("Raw DHT Info: {:?}", info); + DhtLogger::debug(&format!("Raw DHT Info: {:?}", info)); // Add explanation of message types - trace!("Message Types you might see in logs:"); - trace!("- find_node: Looking for nodes close to an ID"); - trace!("- get_peers: Looking for peers for an infohash"); - trace!("- announce_peer: Announcing that it's a peer for an infohash"); - trace!("- ping: Checking if a node is alive"); - trace!("- NoValues: Response indicating no requested values were found"); + DhtLogger::trace("Message Types you might see in logs:"); + DhtLogger::trace("- find_node: Looking for nodes close to an ID"); + DhtLogger::trace("- get_peers: Looking for peers for an infohash"); + DhtLogger::trace("- announce_peer: Announcing that it's a peer for an infohash"); + DhtLogger::trace("- ping: Checking if a node is alive"); + DhtLogger::trace("- NoValues: Response indicating no requested values were found"); - info!(""); // Blank line to separate updates + DhtLogger::info(""); // Blank line to separate updates } } \ No newline at end of file From 67591e58c45d2002d9f141b882dfb323cb686643 Mon Sep 17 00:00:00 2001 From: Miguel Medeiros Date: Sun, 9 Feb 2025 00:16:32 -0300 Subject: [PATCH 3/7] refactor: improve code formatting and readability in node example --- examples/node.rs | 52 ++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/examples/node.rs b/examples/node.rs index 3df9a22a..5b877c71 100644 --- a/examples/node.rs +++ b/examples/node.rs @@ -1,8 +1,8 @@ +use colored::*; use mainline::Dht; use std::{thread, time::Duration}; -use tracing::{Level, info, debug, trace}; +use tracing::{debug, info, trace, Level}; use tracing_subscriber::{self, fmt::format::FmtSpan}; -use colored::*; /// Custom logger that formats DHT messages struct DhtLogger; @@ -71,7 +71,6 @@ fn format_message(msg: &str, direction: ColoredString) -> String { /// Format message details in a structured way fn format_message_details(msg: &str) -> String { let mut details = String::new(); - // Extract and format common fields if let Some(id) = extract_field(msg, "id=") { details.push_str(&format!(" ID: {}\n", id.bright_yellow())); @@ -88,13 +87,14 @@ fn format_message_details(msg: &str) -> String { if let Some(token) = extract_field(msg, "token=") { details.push_str(&format!(" Token: {}\n", token.bright_yellow())); } - details } /// Extract a field from the message fn extract_field<'a>(msg: &'a str, field: &str) -> Option<&'a str> { - msg.split(field).nth(1).map(|s| s.split_whitespace().next().unwrap_or("")) + msg.split(field) + .nth(1) + .map(|s| s.split_whitespace().next().unwrap_or("")) } /// Format nodes list for better readability @@ -105,7 +105,11 @@ fn format_nodes(nodes: &str) -> String { .map(|n| n.bright_cyan().to_string()) .collect::>() .join(", ") - + if nodes.split(", ").count() > 3 { " ..." } else { "" } + + if nodes.split(", ").count() > 3 { + " ..." + } else { + "" + } } fn main() { @@ -126,7 +130,7 @@ fn main() { .with_file(true) .with_line_number(true) .with_target(true) - .compact() // This will make the output more compact + .compact(), // This will make the output more compact ) .init(); @@ -137,44 +141,49 @@ fn main() { .expect("Failed to create DHT server"); DhtLogger::info("DHT server node is running! Press Ctrl+C to stop."); - // Wait for bootstrap to complete DhtLogger::info("Waiting for bootstrap..."); dht.bootstrapped(); DhtLogger::info("Bootstrap complete!"); - // Keep the program running and show periodic information loop { thread::sleep(Duration::from_secs(30)); let info = dht.info(); - // Basic node information DhtLogger::info("=== DHT Node Status ==="); DhtLogger::info(&format!("Node ID: {}", info.id().to_string().yellow())); - DhtLogger::info(&format!("Local address: {}", info.local_addr().to_string().blue())); + DhtLogger::info(&format!( + "Local address: {}", + info.local_addr().to_string().blue() + )); if let Some(addr) = info.public_address() { DhtLogger::info(&format!("Public address: {}", addr.to_string().green())); } DhtLogger::info(&format!( - "Firewalled: {}", - if info.firewalled() { "Yes".red() } else { "No".green() } + "Firewalled: {}", + if info.firewalled() { + "Yes".red() + } else { + "No".green() + } )); DhtLogger::info(&format!( - "Server mode: {}", - if info.server_mode() { "Yes".green() } else { "No".yellow() } + "Server mode: {}", + if info.server_mode() { + "Yes".green() + } else { + "No".yellow() + } )); - // Network statistics let (size_estimate, std_dev) = info.dht_size_estimate(); DhtLogger::info("=== Network Statistics ==="); DhtLogger::info(&format!( - "Estimated nodes in network: {} (±{}%)", + "Estimated nodes in network: {} (±{}%)", size_estimate.to_string().cyan(), format!("{:.1}", std_dev * 100.0).yellow() - )); - + )); DhtLogger::debug(&format!("Raw DHT Info: {:?}", info)); - // Add explanation of message types DhtLogger::trace("Message Types you might see in logs:"); DhtLogger::trace("- find_node: Looking for nodes close to an ID"); @@ -182,7 +191,6 @@ fn main() { DhtLogger::trace("- announce_peer: Announcing that it's a peer for an infohash"); DhtLogger::trace("- ping: Checking if a node is alive"); DhtLogger::trace("- NoValues: Response indicating no requested values were found"); - DhtLogger::info(""); // Blank line to separate updates } -} \ No newline at end of file +} From f8a99e2db4edd177d773acc981e788fe6c9d07fe Mon Sep 17 00:00:00 2001 From: Miguel Medeiros Date: Sun, 9 Feb 2025 00:30:33 -0300 Subject: [PATCH 4/7] refactor: remove unused trace logging from DHT logger --- examples/node.rs | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/examples/node.rs b/examples/node.rs index 5b877c71..fbbf1e8c 100644 --- a/examples/node.rs +++ b/examples/node.rs @@ -1,7 +1,7 @@ use colored::*; use mainline::Dht; use std::{thread, time::Duration}; -use tracing::{debug, info, trace, Level}; +use tracing::{debug, info, Level}; use tracing_subscriber::{self, fmt::format::FmtSpan}; /// Custom logger that formats DHT messages @@ -15,10 +15,6 @@ impl DhtLogger { fn debug(msg: &str) { debug!("{}", format_dht_message(msg)); } - - fn trace(msg: &str) { - trace!("{}", format_dht_message(msg)); - } } /// Format a DHT message for better readability @@ -184,13 +180,6 @@ fn main() { format!("{:.1}", std_dev * 100.0).yellow() )); DhtLogger::debug(&format!("Raw DHT Info: {:?}", info)); - // Add explanation of message types - DhtLogger::trace("Message Types you might see in logs:"); - DhtLogger::trace("- find_node: Looking for nodes close to an ID"); - DhtLogger::trace("- get_peers: Looking for peers for an infohash"); - DhtLogger::trace("- announce_peer: Announcing that it's a peer for an infohash"); - DhtLogger::trace("- ping: Checking if a node is alive"); - DhtLogger::trace("- NoValues: Response indicating no requested values were found"); DhtLogger::info(""); // Blank line to separate updates } } From 31e99a678796d8d102eb0e68bde84e18d958d46e Mon Sep 17 00:00:00 2001 From: Miguel Medeiros Date: Sun, 9 Feb 2025 08:40:33 -0300 Subject: [PATCH 5/7] fix: move colored dependency to dev dependencies --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index b0716872..ac9f4662 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,6 @@ ed25519-dalek = "2.1.1" tracing = "0.1" lru = { version = "0.12.5", default-features = false } dyn-clone = "1.0.18" -colored = "2.0.0" document-features = "0.2.10" @@ -42,6 +41,7 @@ histo = "1.0.0" rayon = "1.10" dashmap = "6.1" flume = "0.11.1" +colored = "2.0.0" [features] ## Include [Dht] node. From 90ea61b3a7900ec60fc483bfd5dc6a468bbaf3a5 Mon Sep 17 00:00:00 2001 From: Miguel Medeiros Date: Sun, 9 Feb 2025 09:02:30 -0300 Subject: [PATCH 6/7] refactor: reorganize examples in README for clarity and structure --- examples/README.md | 83 ++++++++++++++++------------------------------ 1 file changed, 28 insertions(+), 55 deletions(-) diff --git a/examples/README.md b/examples/README.md index 6e2baf07..79401736 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,85 +1,58 @@ # Examples -## Announce peer +## Core API Examples +These examples demonstrate the main functionality of the Mainline DHT library: +### Basic DHT Operations ```sh -cargo run --example announce_peer <40 bytes hex info_hash> -``` +# Bootstrap a DHT node +cargo run --example bootstrap -## Get peers +# Implement a custom request filter +cargo run --example request_filter +# Cache and reuse bootstrap nodes +cargo run --example cache_bootstrap +``` + +### Peer Operations ```sh +# Announce as a peer +cargo run --example announce_peer <40 bytes hex info_hash> + +# Find peers cargo run --example get_peers <40 bytes hex info_hash> ``` -## Put Immutable - +### Data Storage ```sh +# Store immutable data cargo run --example put_immutable -``` -## Get Immutable - -```sh +# Retrieve immutable data cargo run --example get_immutable <40 bytes hex target from put_immutable> -``` -## Put Mutable - -```sh +# Store mutable data cargo run --example put_mutable <64 bytes hex secret_key> -``` -## Get Mutable - -```sh +# Retrieve mutable data cargo run --example get_mutable <40 bytes hex target from put_mutable> ``` -## Request Filter - -Example showing how to implement a custom request filter for the DHT server: - -```sh -cargo run --example request_filter -``` - -## Cache Bootstrap +--- -Example demonstrating how to cache and reuse bootstrapping nodes: +## Analysis & Research Tools +These examples are for DHT network analysis and research purposes: -```sh -cargo run --example cache_bootstrap -``` - -## Count IPs Close to Key - -Count all IP addresses around a random target ID and analyze hit rates: +> Note: These tools are not part of the main API and are provided for curiosity/research only. ```sh +# Analyze DHT node distribution cargo run --example count_ips_close_to_key -``` - -## Mark Recapture DHT - -Estimate DHT size using Mark-Recapture method: -```sh +# Estimate DHT size (Mark-Recapture method) cargo run --example mark_recapture_dht -``` - -## Measure DHT -Measure the size of the DHT network: - -```sh +# Measure DHT network size cargo run --example measure_dht ``` - -## Bootstrap - -Basic example of bootstrapping a DHT node: - -```sh -cargo run --example bootstrap -``` From bfa3ab21b2956badd6bb192f643a7501c4c6ab59 Mon Sep 17 00:00:00 2001 From: Miguel Medeiros Date: Sun, 9 Feb 2025 09:05:42 -0300 Subject: [PATCH 7/7] fix: renamed example node.rs to logging.rs --- examples/README.md | 3 +++ examples/{node.rs => logging.rs} | 0 2 files changed, 3 insertions(+) rename examples/{node.rs => logging.rs} (100%) diff --git a/examples/README.md b/examples/README.md index 79401736..5d692910 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,6 +13,9 @@ cargo run --example request_filter # Cache and reuse bootstrap nodes cargo run --example cache_bootstrap + +# Advanced logging configuration +cargo run --example logging ``` ### Peer Operations diff --git a/examples/node.rs b/examples/logging.rs similarity index 100% rename from examples/node.rs rename to examples/logging.rs