From d9c24e062f36ec9b7d9963764186808f3359235a Mon Sep 17 00:00:00 2001 From: Naman Garg <0708ng@gmail.com> Date: Sun, 14 Jul 2024 16:29:23 +0530 Subject: [PATCH 01/11] chore: add run id --- benchmark/remote.py | 41 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/benchmark/remote.py b/benchmark/remote.py index bd768bf..6a11e90 100644 --- a/benchmark/remote.py +++ b/benchmark/remote.py @@ -416,29 +416,6 @@ def _logs(self, hosts, faults, committee=[]): #, servers, run_id): logParser.log_parser(self.mechanism.name, PathMaker.logs_path(), faults=faults) return logParser - # # Delete local logs (if any). - # cmd = CommandMaker.clean_logs() - # subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL) - - # hosts_df = pd.DataFrame(columns=['ip', 'node_num']) - - # # Download log files. - # progress = progress_bar(hosts, prefix='Downloading logs:') - # for i, host in enumerate(progress): - # c = Connection(host, user='ubuntu', connect_kwargs=self.connect) - # c.get(PathMaker.node_log_file(i), local=PathMaker.node_log_file(i)) - # c.get( - # PathMaker.client_log_file(i), local=PathMaker.client_log_file(i) - # ) - # # mapping HOST <---> i - # new_data = pd.DataFrame({'ip':host, 'node_num':i}, index=[0]) - # hosts_df = pd.concat([hosts_df, new_data], ignore_index = True) - - # servers = pd.merge(servers, hosts_df, on='ip') - # # Parse logs and return the parser. - # Print.info('Parsing logs and computing performance...') - # return LogParser.process(PathMaker.logs_path(), faults=faults, servers=servers, run_id =run_id) - def run(self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=False): assert isinstance(debug, bool) Print.heading(f'Starting {self.mechanism.name} remote benchmark') @@ -517,23 +494,25 @@ def run(self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=Fa faults = bench_parameters.faults hosts = hosts[:n-faults] - # run_id_array = [] + run_id_array = [] # Run the benchmark. for i in range(bench_parameters.runs): - Print.heading(f'Run {i+1}/{bench_parameters.runs}') - # run_id = GeoLogParser.get_new_run_id() - # Print.heading(f'Run {i+1}/{bench_parameters.runs} with run_id {run_id}') + run_id = GeoLogParser.get_new_run_id() + Print.heading(f'Run {i+1}/{bench_parameters.runs} with run_id {run_id}') + try: self._run_single( hosts, r, bench_parameters, node_parameters, debug, committee_copy ) + logger = self._logs(hosts, faults, committee_copy) print(logger.result()) logger.print(PathMaker.result_file( self.mechanism.name, n, r, bench_parameters.tx_size, faults )) - # run_id_array.append(run_id) + + run_id_array.append(run_id) except (subprocess.SubprocessError, GroupException, ParseError) as e: self.kill(hosts=hosts) if isinstance(e, GroupException): @@ -541,9 +520,9 @@ def run(self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=Fa Print.error(BenchError('Benchmark failed', e)) continue - # aggregated_results = GeoLogParser.aggregate_runs(run_id_array) - # print(aggregated_results) - # aggregated_results.to_csv('/home/ubuntu/results/64node-fixed-mean-geo-dec-metrics.csv', mode='a', index=False, header=False) + aggregated_results = GeoLogParser.aggregate_runs(run_id_array) + print(aggregated_results) + aggregated_results.to_csv('/home/ubuntu/results/64node-fixed-mean-geo-dec-metrics.csv', mode='a', index=False, header=False) if isGeoRemote: # Delete delay parameters. From c89248317648ce76c444e4fd403cf09feb6d2aac Mon Sep 17 00:00:00 2001 From: Naman Garg <0708ng@gmail.com> Date: Sun, 14 Jul 2024 16:39:11 +0530 Subject: [PATCH 02/11] chore: format --- benchmark/remote.py | 380 ++++++++++++++++++++++++++------------------ 1 file changed, 228 insertions(+), 152 deletions(-) diff --git a/benchmark/remote.py b/benchmark/remote.py index 6a11e90..91b9d62 100644 --- a/benchmark/remote.py +++ b/benchmark/remote.py @@ -1,31 +1,45 @@ -from fabric import Connection, ThreadingGroup as Group -from fabric.exceptions import GroupException -from paramiko import RSAKey -from paramiko.ssh_exception import PasswordRequiredException, SSHException -from os.path import basename, splitext -from time import sleep -from math import ceil -from collections import OrderedDict import csv import subprocess +from collections import OrderedDict from copy import deepcopy -# import pandas as pd +from math import ceil +from os.path import basename, splitext +from time import sleep + +from fabric import Connection +from fabric import ThreadingGroup as Group +from fabric.exceptions import GroupException +from paramiko import RSAKey +from paramiko.ssh_exception import PasswordRequiredException, SSHException -from benchmark.config import Committee, Key, NodeParameters, BenchParameters, ConfigError -from benchmark.utils import BenchError, Print, PathMaker, progress_bar, set_weight from benchmark.commands import CommandMaker -from benchmark.instance import InstanceManager -from benchmark.geodec import GeoDec +from benchmark.config import ( + BenchParameters, + Committee, + ConfigError, + Key, + NodeParameters, +) from benchmark.geo_logs import GeoLogParser +from benchmark.geodec import GeoDec +from benchmark.instance import InstanceManager from benchmark.latency_setter import LatencySetter - +from benchmark.logs import LogParser, ParseError +from benchmark.mechanisms.bullshark import ( + BullsharkBenchParameters, + BullsharkCommittee, + BullsharkMechanism, + BullsharkNodeParameters, +) from benchmark.mechanisms.cometbft import CometBftMechanism from benchmark.mechanisms.hotstuff import HotStuffMechanism -from benchmark.mechanisms.bullshark import BullsharkMechanism, BullsharkBenchParameters, BullsharkNodeParameters, BullsharkCommittee -from benchmark.logs import LogParser, ParseError +from benchmark.utils import BenchError, PathMaker, Print, progress_bar, set_weight + +# import pandas as pd + class FabricError(Exception): - ''' Wrapper for Fabric exception with a meaningfull error message. ''' + """Wrapper for Fabric exception with a meaningfull error message.""" def __init__(self, error): assert isinstance(error, GroupException) @@ -41,7 +55,7 @@ class Bench: def __init__(self, ctx, mechanism): consensusMechanisms = ["cometbft", "hotstuff", "bullshark"] if mechanism not in consensusMechanisms: - raise BenchError('Consensus mechanism support not available', e) + raise BenchError("Consensus mechanism support not available", e) self.manager = InstanceManager.make(mechanism) self.settings = self.manager.settings @@ -59,7 +73,7 @@ def __init__(self, ctx, mechanism): ) self.connect = ctx.connect_kwargs except (IOError, PasswordRequiredException, SSHException) as e: - raise BenchError('Failed to load SSH key', e) + raise BenchError("Failed to load SSH key", e) def _check_stderr(self, output): if isinstance(output, dict): @@ -72,44 +86,44 @@ def _check_stderr(self, output): raise ExecutionError(output.stderr) def install(self): - Print.info(f'Installing {self.settings.testbed}') + Print.info(f"Installing {self.settings.testbed}") cmd = self.mechanism.install_cmd hosts = self._select_hosts() - + try: g = Group(*hosts, user=self.settings.key_name, connect_kwargs=self.connect) - g.run(' && '.join(cmd), hide=True) - Print.heading(f'Initialized testbed of {len(hosts)} nodes') + g.run(" && ".join(cmd), hide=True) + Print.heading(f"Initialized testbed of {len(hosts)} nodes") except (GroupException, ExecutionError) as e: e = FabricError(e) if isinstance(e, GroupException) else e - raise BenchError('Failed to install repo on testbed', e) + raise BenchError("Failed to install repo on testbed", e) def kill(self, hosts=[], delete_logs=False): assert isinstance(hosts, list) assert isinstance(delete_logs, bool) hosts = self._select_hosts([len(hosts)]) - delete_logs = CommandMaker.clean_logs() if delete_logs else 'true' - cmd = [delete_logs, f'({CommandMaker.kill()} || true)'] + delete_logs = CommandMaker.clean_logs() if delete_logs else "true" + cmd = [delete_logs, f"({CommandMaker.kill()} || true)"] try: g = Group(*hosts, user=self.settings.key_name, connect_kwargs=self.connect) - g.run(' && '.join(cmd), hide=True) + g.run(" && ".join(cmd), hide=True) except GroupException as e: - raise BenchError('Failed to kill nodes', FabricError(e)) + raise BenchError("Failed to kill nodes", FabricError(e)) def _select_hosts(self, nodes=[]): max_count = max(nodes) - - addrs = [] + + addrs = [] # Retrieve values based on your scripts, note we use Internal IP addresses - with open(self.settings.ip_file, 'r') as f: - # If you used the GCP scripts from here https://github.com/sm86/gcp-scripts - if(self.settings.provider == "google_compute_engine"): + with open(self.settings.ip_file, "r") as f: + # If you used the GCP scripts from here https://github.com/sm86/gcp-scripts + if self.settings.provider == "google_compute_engine": reader = csv.DictReader(f) for row in reader: - addrs.append(row['Internal IP']) + addrs.append(row["Internal IP"]) else: - addrs = [line.strip() for line in f.readlines()] + addrs = [line.strip() for line in f.readlines()] return addrs[:max_count] def _background_run(self, host, command, log_file): @@ -120,27 +134,27 @@ def _background_run(self, host, command, log_file): self._check_stderr(output) def _update(self, hosts): - Print.info( - f'Updating {len(hosts)} nodes (branch "{self.settings.branch}")...' - ) - + Print.info(f'Updating {len(hosts)} nodes (branch "{self.settings.branch}")...') + cmd = self.mechanism.update_cmd - + g = Group(*hosts, user=self.settings.key_name, connect_kwargs=self.connect) - g.run(' && '.join(cmd), hide=True) + g.run(" && ".join(cmd), hide=True) def _config(self, isGeoremote, hosts, node_parameters, bench_parameters=None): - Print.info('Generating configuration files...') - + Print.info("Generating configuration files...") + # Cleanup all local configuration files. cmd = CommandMaker.cleanup() subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL) - - if self.mechanism.name == 'cometbft': + + if self.mechanism.name == "cometbft": # Cleanup node configuration files on hosts for i, host in enumerate(hosts): cmd = CommandMaker.clean_node_config(i) - c = Connection(host, user=self.settings.key_name, connect_kwargs=self.connect) + c = Connection( + host, user=self.settings.key_name, connect_kwargs=self.connect + ) c.run(cmd, shell=True) # Create persistent peers @@ -148,39 +162,45 @@ def _config(self, isGeoremote, hosts, node_parameters, bench_parameters=None): hosts_string = " ".join(hosts) - with open('persistent_peer.txt', 'w') as f: + with open("persistent_peer.txt", "w") as f: f.write("") f.close() # Create testnet config files - cmd = [f'~/cometbft testnet --v {len(hosts)}'] + cmd = [f"~/cometbft testnet --v {len(hosts)}"] # cmd = [f'~/cometbft testnet --v {len(hosts)} --config ~/geodec/testdata/cometbft-config.toml'] # NOTE custom configuration subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL) - + # Update the stake weights in the configuration file if isGeoremote: set_weight(self.mechanism.name, self.settings.geo_input) - + # Run the bash file and store the ouput in this file cmd = [ # 'chmod u+x ./persistent.sh', - f'./persistent.sh {hosts_string}' + f"./persistent.sh {hosts_string}" ] subprocess.run(cmd, shell=True) - + # Upload configuration files. - progress = progress_bar(hosts, prefix='Uploading config files:') + progress = progress_bar(hosts, prefix="Uploading config files:") for i, host in enumerate(hosts): - cmd = [f'scp -i {self.settings.key_path} -r ~/geodec/mytestnet/node{i} ubuntu@{host}:~/'] # NOTE Path of the node config files + cmd = [ + f"scp -i {self.settings.key_path} -r ~/geodec/mytestnet/node{i} ubuntu@{host}:~/" + ] # NOTE Path of the node config files subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL) - + else: # Recompile the latest code. cmd = CommandMaker.compile().split() - subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path(self.settings.repo_name)) + subprocess.run( + cmd, check=True, cwd=PathMaker.node_crate_path(self.settings.repo_name) + ) # Create alias for the client and nodes binary. - cmd = CommandMaker.alias_binaries(PathMaker.binary_path(self.settings.repo_name), self.mechanism.name) + cmd = CommandMaker.alias_binaries( + PathMaker.binary_path(self.settings.repo_name), self.mechanism.name + ) subprocess.run([cmd], shell=True) # Generate configuration files. @@ -190,50 +210,57 @@ def _config(self, isGeoremote, hosts, node_parameters, bench_parameters=None): cmd = CommandMaker.generate_key(filename, self.mechanism.name).split() subprocess.run(cmd, check=True) keys += [Key.from_file(filename)] - names = [x.name for x in keys] - - if self.mechanism.name == 'hotstuff': - consensus_addr = [f'{x}:{self.settings.ports["consensus"]}' for x in hosts] + + if self.mechanism.name == "hotstuff": + consensus_addr = [ + f'{x}:{self.settings.ports["consensus"]}' for x in hosts + ] front_addr = [f'{x}:{self.settings.ports["front"]}' for x in hosts] mempool_addr = [f'{x}:{self.settings.ports["mempool"]}' for x in hosts] committee = Committee(names, consensus_addr, front_addr, mempool_addr) - elif self.mechanism.name == 'bullshark': + elif self.mechanism.name == "bullshark": if bench_parameters.collocate: workers = bench_parameters.workers - addresses = OrderedDict((x, [y] * (workers + 1)) for x, y in zip(names, hosts)) + addresses = OrderedDict( + (x, [y] * (workers + 1)) for x, y in zip(names, hosts) + ) else: addresses = OrderedDict((x, y) for x, y in zip(names, hosts)) committee = BullsharkCommittee(addresses, self.settings.ports["base"]) committee.print(PathMaker.committee_file()) node_parameters.print(PathMaker.parameters_file()) - + if isGeoremote: set_weight(self.mechanism.name, self.settings.geo_input) - - cmd = f'{CommandMaker.cleanup()} || true' + + cmd = f"{CommandMaker.cleanup()} || true" g = Group(*hosts, user=self.settings.key_name, connect_kwargs=self.connect) g.run(cmd, hide=True) # NOTE Upload configuration files. - progress = progress_bar(hosts, prefix='Uploading config files:') + progress = progress_bar(hosts, prefix="Uploading config files:") for i, host in enumerate(progress): - c = Connection(host, user=self.settings.key_name, connect_kwargs=self.connect) - c.put(PathMaker.committee_file(), '.') - c.put(PathMaker.key_file(i), '.') - c.put(PathMaker.parameters_file(), '.') + c = Connection( + host, user=self.settings.key_name, connect_kwargs=self.connect + ) + c.put(PathMaker.committee_file(), ".") + c.put(PathMaker.key_file(i), ".") + c.put(PathMaker.parameters_file(), ".") return committee - def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=False, committee=[]): - Print.info('Booting testbed...') + def _run_single( + self, hosts, rate, bench_parameters, node_parameters, debug=False, committee=[] + ): + Print.info("Booting testbed...") # Kill any potentially unfinished run and delete logs. self.kill(hosts=hosts, delete_logs=True) - if self.mechanism.name == 'hotstuff': + if self.mechanism.name == "hotstuff": # Run the clients (they will wait for the nodes to be ready). # Filter all faulty nodes from the client addresses (or they will wait # for the faulty nodes to be online). @@ -249,7 +276,7 @@ def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=Fals rate_share, self.mechanism.name, timeout, - nodes=addresses + nodes=addresses, ) self._background_run(host, cmd, log_file) @@ -264,27 +291,29 @@ def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=Fals db, PathMaker.parameters_file(), debug=debug, - mechanism=self.mechanism.name + mechanism=self.mechanism.name, ) self._background_run(host, cmd, log_file) - + # Wait for the nodes to synchronize - Print.info('Waiting for the nodes to synchronize...') + Print.info("Waiting for the nodes to synchronize...") sleep(2 * node_parameters.timeout_delay / 1000) - elif self.mechanism.name == 'cometbft': + elif self.mechanism.name == "cometbft": persistent_peers = [] - - with open('persistent_peer.txt', 'r') as f: + + with open("persistent_peer.txt", "r") as f: persistent_peers = f.read() persistent_peers = persistent_peers[:-1] - + # Run the clients # committee = Committee.load(PathMaker.committee_file()) # TODO for cometbft addresses = [f'{x}:{self.settings.ports["front"]}' for x in hosts] # rate_share = ceil(rate / committee.size()) # TODO Take faults into account. rate_share = ceil(rate / len(hosts)) - duration = bench_parameters.duration # Duration for which the client should run + duration = ( + bench_parameters.duration + ) # Duration for which the client should run client_logs = [PathMaker.client_log_file(i) for i in range(len(hosts))] for host, addr, log_file in zip(hosts, addresses, client_logs): cmd = CommandMaker.run_client( @@ -293,7 +322,7 @@ def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=Fals rate_share, self.mechanism.name, duration, - nodes=addresses + nodes=addresses, ) self._background_run(host, cmd, log_file) @@ -303,16 +332,16 @@ def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=Fals cmd = f'./node node --home ~/node{i} --proxy_app=kvstore --p2p.persistent_peers="{persistent_peers}" --log_level="state:info,consensus:info,txindex:info,consensus:debug,*:error"' self._background_run(host, cmd, log_file) - elif self.mechanism.name == 'bullshark': + elif self.mechanism.name == "bullshark": faults = bench_parameters.faults # Run the clients (they will wait for the nodes to be ready). # Filter all faulty nodes from the client addresses (or they will wait # for the faulty nodes to be online). - Print.info('Booting clients...') + Print.info("Booting clients...") workers_addresses = committee.workers_addresses(faults) rate_share = ceil(rate / committee.workers()) for i, addresses in enumerate(workers_addresses): - for (id, address) in addresses: + for id, address in addresses: host = BullsharkCommittee.ip(address) cmd = CommandMaker.run_client( address, @@ -326,7 +355,7 @@ def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=Fals self._background_run(host, cmd, log_file) # Run the primaries (except the faulty ones). - Print.info('Booting primaries...') + Print.info("Booting primaries...") for i, address in enumerate(committee.primary_addresses(faults)): host = BullsharkCommittee.ip(address) cmd = CommandMaker.run_primary( @@ -334,15 +363,15 @@ def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=Fals PathMaker.committee_file(), PathMaker.db_path(i), PathMaker.parameters_file(), - debug=debug + debug=debug, ) log_file = PathMaker.primary_log_file(i) self._background_run(host, cmd, log_file) # Run the workers (except the faulty ones). - Print.info('Booting workers...') + Print.info("Booting workers...") for i, addresses in enumerate(workers_addresses): - for (id, address) in addresses: + for id, address in addresses: host = BullsharkCommittee.ip(address) cmd = CommandMaker.run_worker( PathMaker.key_file(i), @@ -350,75 +379,84 @@ def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=Fals PathMaker.db_path(i, id), PathMaker.parameters_file(), id, # The worker's id. - debug=debug + debug=debug, ) log_file = PathMaker.worker_log_file(i, id) self._background_run(host, cmd, log_file) # Wait for all transactions to be processed. duration = bench_parameters.duration - for _ in progress_bar(range(20), prefix=f'Running benchmark ({duration} sec):'): + for _ in progress_bar(range(20), prefix=f"Running benchmark ({duration} sec):"): sleep(ceil(duration / 20)) self.kill(hosts=hosts, delete_logs=False) - + sleep(1) - if self.mechanism.name == 'cometbft': + if self.mechanism.name == "cometbft": latency_logs = [PathMaker.latency_log_file(i) for i in range(len(hosts))] for i, (host, log_file) in enumerate(zip(hosts, latency_logs)): - cmd = f'./cometbft/test/loadtime/build/report --database-type goleveldb --data-dir ~/node{i}/data' + cmd = f"./cometbft/test/loadtime/build/report --database-type goleveldb --data-dir ~/node{i}/data" self._background_run(host, cmd, log_file) - - def _logs(self, hosts, faults, committee=[]): #, servers, run_id): + def _logs(self, hosts, faults, committee=[]): # , servers, run_id): # Delete local logs (if any). cmd = CommandMaker.clean_logs() subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL) # Download log files. - progress = progress_bar(hosts, prefix='Downloading logs:') + progress = progress_bar(hosts, prefix="Downloading logs:") if self.mechanism.name == "bullshark": workers_addresses = committee.workers_addresses(faults) - progress = progress_bar(workers_addresses, prefix='Downloading workers logs:') + progress = progress_bar( + workers_addresses, prefix="Downloading workers logs:" + ) for i, addresses in enumerate(progress): for id, address in addresses: host = BullsharkCommittee.ip(address) - c = Connection(host, user='ubuntu', connect_kwargs=self.connect) + c = Connection(host, user="ubuntu", connect_kwargs=self.connect) c.get( - PathMaker.client_log_file_bull(i, id), - local=PathMaker.client_log_file_bull(i, id) + PathMaker.client_log_file_bull(i, id), + local=PathMaker.client_log_file_bull(i, id), ) c.get( - PathMaker.worker_log_file(i, id), - local=PathMaker.worker_log_file(i, id) + PathMaker.worker_log_file(i, id), + local=PathMaker.worker_log_file(i, id), ) primary_addresses = committee.primary_addresses(faults) - progress = progress_bar(primary_addresses, prefix='Downloading primaries logs:') + progress = progress_bar( + primary_addresses, prefix="Downloading primaries logs:" + ) for i, address in enumerate(progress): host = BullsharkCommittee.ip(address) - c = Connection(host, user='ubuntu', connect_kwargs=self.connect) + c = Connection(host, user="ubuntu", connect_kwargs=self.connect) c.get( - PathMaker.primary_log_file(i), - local=PathMaker.primary_log_file(i) + PathMaker.primary_log_file(i), local=PathMaker.primary_log_file(i) ) else: for i, host in enumerate(progress): - c = Connection(host, user=self.settings.key_name, connect_kwargs=self.connect) + c = Connection( + host, user=self.settings.key_name, connect_kwargs=self.connect + ) c.get(PathMaker.node_log_file(i), local=PathMaker.node_log_file(i)) c.get(PathMaker.client_log_file(i), local=PathMaker.client_log_file(i)) if self.mechanism.name == "cometbft": - c.get(PathMaker.latency_log_file(i), local=PathMaker.latency_log_file(i)) - + c.get( + PathMaker.latency_log_file(i), + local=PathMaker.latency_log_file(i), + ) + # Parse logs and return the parser. - Print.info('Parsing logs and computing performance...') + Print.info("Parsing logs and computing performance...") logParser = LogParser() logParser.log_parser(self.mechanism.name, PathMaker.logs_path(), faults=faults) return logParser - def run(self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=False): + def run( + self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=False + ): assert isinstance(debug, bool) - Print.heading(f'Starting {self.mechanism.name} remote benchmark') + Print.heading(f"Starting {self.mechanism.name} remote benchmark") try: if self.mechanism.name == "bullshark": @@ -428,12 +466,12 @@ def run(self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=Fa bench_parameters = BenchParameters(bench_parameters_dict) node_parameters = NodeParameters(node_parameters_dict) except ConfigError as e: - raise BenchError('Invalid nodes or bench parameters', e) - + raise BenchError("Invalid nodes or bench parameters", e) + # Select which hosts to use. selected_hosts = self._select_hosts(bench_parameters.nodes) if len(selected_hosts) < max(bench_parameters.nodes): - Print.warn('There are not enough instances available') + Print.warn("There are not enough instances available") return # Update nodes. @@ -441,50 +479,66 @@ def run(self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=Fa self._update(selected_hosts) except (GroupException, ExecutionError) as e: e = FabricError(e) if isinstance(e, GroupException) else e - raise BenchError('Failed to update nodes', e) - + raise BenchError("Failed to update nodes", e) + if isGeoRemote: geo_input = GeoDec.getGeoInput(self.settings.geo_input) - selected_servers = GeoDec.getAllServers(geo_input, self.settings.servers_file, self.settings.ip_file) - pingDelays = GeoDec.getPingDelay(geo_input, self.settings.ping_grouped_file, self.settings.pings_file) - + selected_servers = GeoDec.getAllServers( + geo_input, self.settings.servers_file, self.settings.ip_file + ) + pingDelays = GeoDec.getPingDelay( + geo_input, self.settings.ping_grouped_file, self.settings.pings_file + ) + Print.heading("\nSelected servers:") - print(selected_servers[['ip', 'id', 'name', 'latitude', 'longitude']].to_string(index=False)) + print( + selected_servers[ + ["ip", "id", "name", "latitude", "longitude"] + ].to_string(index=False) + ) Print.heading("\nPing Delays:") - print(pingDelays[['source', 'destination', 'avg', 'mdev']].to_string(index=False)) - + print( + pingDelays[["source", "destination", "avg", "mdev"]].to_string( + index=False + ) + ) + if len(pingDelays) != len(selected_servers) * (len(selected_servers) - 1): - print('ERROR: Ping delays not available for all servers') + print("ERROR: Ping delays not available for all servers") return - + # Set delay parameters. latencySetter = LatencySetter(self.settings, self.connect) try: latencySetter.deleteDelay(selected_hosts) except: pass - + try: latencySetter.configDelay(selected_hosts) - latencySetter.addDelays(selected_servers, pingDelays, self.settings.interface) + latencySetter.addDelays( + selected_servers, pingDelays, self.settings.interface + ) except (subprocess.SubprocessError, GroupException) as e: e = FabricError(e) if isinstance(e, GroupException) else e - Print.error(BenchError('Failed to initalize delays', e)) - + Print.error(BenchError("Failed to initalize delays", e)) + # Run benchmarks. for n in bench_parameters.nodes: for r in bench_parameters.rate: - Print.heading(f'\nRunning {n} nodes (input rate: {r:,} tx/s)') + Print.heading(f"\nRunning {n} nodes (input rate: {r:,} tx/s)") hosts = selected_hosts[:n] # Upload all configuration files. try: - committee = self._config(isGeoRemote, hosts, node_parameters, bench_parameters) + committee = self._config( + isGeoRemote, hosts, node_parameters, bench_parameters + ) except (subprocess.SubprocessError, GroupException) as e: e = FabricError(e) if isinstance(e, GroupException) else e - Print.error(BenchError('Failed to configure nodes', e)) + Print.error(BenchError("Failed to configure nodes", e)) continue - + committee_copy = [] if self.mechanism.name == "bullshark": committee_copy = deepcopy(committee) @@ -492,37 +546,59 @@ def run(self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=Fa # Do not boot faulty nodes. faults = bench_parameters.faults - hosts = hosts[:n-faults] - + hosts = hosts[: n - faults] + run_id_array = [] - + # Run the benchmark. for i in range(bench_parameters.runs): run_id = GeoLogParser.get_new_run_id() - Print.heading(f'Run {i+1}/{bench_parameters.runs} with run_id {run_id}') - + Print.heading( + f"Run {i+1}/{bench_parameters.runs} with run_id {run_id}" + ) + try: self._run_single( - hosts, r, bench_parameters, node_parameters, debug, committee_copy + hosts, + r, + bench_parameters, + node_parameters, + debug, + committee_copy, ) - + logger = self._logs(hosts, faults, committee_copy) print(logger.result()) - logger.print(PathMaker.result_file( - self.mechanism.name, n, r, bench_parameters.tx_size, faults - )) - + logger.print( + PathMaker.result_file( + self.mechanism.name, + n, + r, + bench_parameters.tx_size, + faults, + ) + ) + run_id_array.append(run_id) - except (subprocess.SubprocessError, GroupException, ParseError) as e: + except ( + subprocess.SubprocessError, + GroupException, + ParseError, + ) as e: self.kill(hosts=hosts) if isinstance(e, GroupException): e = FabricError(e) - Print.error(BenchError('Benchmark failed', e)) + Print.error(BenchError("Benchmark failed", e)) continue - + aggregated_results = GeoLogParser.aggregate_runs(run_id_array) print(aggregated_results) - aggregated_results.to_csv('/home/ubuntu/results/64node-fixed-mean-geo-dec-metrics.csv', mode='a', index=False, header=False) + aggregated_results.to_csv( + "/home/ubuntu/results/64node-fixed-mean-geo-dec-metrics.csv", + mode="a", + index=False, + header=False, + ) if isGeoRemote: # Delete delay parameters. @@ -531,4 +607,4 @@ def run(self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=Fa latencySetter.deleteDelay(selected_hosts) except (subprocess.SubprocessError, GroupException) as e: e = FabricError(e) if isinstance(e, GroupException) else e - Print.error(BenchError('Failed to initalize delays', e)) \ No newline at end of file + Print.error(BenchError("Failed to initalize delays", e)) From d0420aad6c83a185baddfd33f7cdb63fac7c6577 Mon Sep 17 00:00:00 2001 From: Naman Garg <0708ng@gmail.com> Date: Sun, 14 Jul 2024 17:21:23 +0530 Subject: [PATCH 03/11] chore: formatter line length 150 --- benchmark/geo_logs.py | 87 ++++++++++++++++++++-------------------- benchmark/remote.py | 92 +++++++++++-------------------------------- 2 files changed, 67 insertions(+), 112 deletions(-) diff --git a/benchmark/geo_logs.py b/benchmark/geo_logs.py index 16de494..2279f2c 100644 --- a/benchmark/geo_logs.py +++ b/benchmark/geo_logs.py @@ -1,12 +1,13 @@ +import math from glob import glob from os.path import join from re import findall, split -import math import pandas as pd from benchmark.utils import PathMaker + ######################################################################################### ######################################################################################### #### GeoDec emulator to study impacts of geospatial diversity on blockchain networks #### @@ -21,75 +22,77 @@ def count_votes_props(run_id): proposals = [] node_num = [] total_block_commits = 0 - for filename in sorted(glob(join(directory, 'node-*.log'))): - node_num.append(int(filename.split('-')[1].split('.')[0])) - with open(filename, 'r') as f: + for filename in sorted(glob(join(directory, "node-*.log"))): + node_num.append(int(filename.split("-")[1].split(".")[0])) + with open(filename, "r") as f: data = f.read() - addr_line = findall(r'Node .* successfully booted', data) - addr = split(' ', addr_line[0])[1] + addr_line = findall(r"Node .* successfully booted", data) + addr = split(" ", addr_line[0])[1] addresses.append(addr) - prop = findall(r'\[(.*Z) .* Created B\d+ -> ([^ ]+=)', data) + prop = findall(r"\[(.*Z) .* Created B\d+ -> ([^ ]+=)", data) proposals.append(len(prop)) - commits = findall(r'QC for block: Round:\d.*', data) + commits = findall(r"QC for block: Round:\d.*", data) total_block_commits = max(total_block_commits, len(commits)) - + votes = [0] * len(addresses) - - for filename in sorted(glob(join(directory, 'node-*.log'))): + + for filename in sorted(glob(join(directory, "node-*.log"))): votes_temp = [0] * len(addresses) - with open(filename, 'r') as f: + with open(filename, "r") as f: logs = f.read() - qc_lines = findall(r'QC for block: Round:\d.*', logs) + qc_lines = findall(r"QC for block: Round:\d.*", logs) for n in range(len(qc_lines)): line = qc_lines[n] for i in range(len(addresses)): if addresses[i] in line: votes_temp[i] = votes_temp[i] + 1 votes[i] = max(votes[i], votes_temp[i]) - + votes_data = pd.DataFrame( - {'address': addresses, - 'votes': votes, - 'proposals': proposals, - 'node_num' : node_num, - 'run_id' : ([run_id] * len(addresses)) - }) + { + "address": addresses, + "votes": votes, + "proposals": proposals, + "node_num": node_num, + "run_id": ([run_id] * len(addresses)), + } + ) return GeoLogParser._calculate_liveliness(votes_data, total_block_commits) - + @staticmethod def _calculate_liveliness(data, total_block_commits): committe_size = len(data) - total_committed = (data['votes'].sum()/ math.ceil((2/3)*committe_size)) - + total_committed = data["votes"].sum() / math.ceil((2 / 3) * committe_size) + total_blocks = max(total_committed, total_block_commits) - - data['liveliness'] = (data['votes']/total_blocks) * 100 - - total_props = data['proposals'].sum() - data['liveliness_woprops'] = ((data['votes']+data['proposals'])/total_props) * 100 + + data["liveliness"] = (data["votes"] / total_blocks) * 100 + + total_props = data["proposals"].sum() + data["liveliness_woprops"] = ((data["votes"] + data["proposals"]) / total_props) * 100 return data - + @staticmethod def get_new_run_id(): - data = pd.read_csv('/home/ubuntu/results/geo-dec-metrics.csv') - return (data['run_id'].max() + 1) - + data = pd.read_csv("/home/ubuntu/results/metrics.csv") + return data["run_id"].max() + 1 + @staticmethod def aggregate_runs(run_id_array): - data = pd.read_csv('/home/ubuntu/results/geo-dec-metrics.csv') - - data = data.loc[data['run_id'].isin(run_id_array)] - by_name = data.groupby(['name']) + data = pd.read_csv("/home/ubuntu/results/geo-dec-metrics.csv") + + data = data.loc[data["run_id"].isin(run_id_array)] + by_name = data.groupby(["name"]) # for name, liveliness in by_name: # print(f"entries for {name!r}") # print("------------------------") # print(liveliness.head(3), end="\n\n") - liveliness_mean = by_name['liveliness'].mean(numeric_only= True).reset_index() - liveliness_mean.rename(columns = {'liveliness':'liveliness_avg'}, inplace = True) + liveliness_mean = by_name["liveliness"].mean(numeric_only=True).reset_index() + liveliness_mean.rename(columns={"liveliness": "liveliness_avg"}, inplace=True) - data_first = data.loc[data['run_id'] == run_id_array[0]] - result = pd.merge(data_first, liveliness_mean, on='name') - result['runs'] = ([len(run_id_array)] * len(result)) - return result \ No newline at end of file + data_first = data.loc[data["run_id"] == run_id_array[0]] + result = pd.merge(data_first, liveliness_mean, on="name") + result["runs"] = [len(run_id_array)] * len(result) + return result diff --git a/benchmark/remote.py b/benchmark/remote.py index 91b9d62..5f87c28 100644 --- a/benchmark/remote.py +++ b/benchmark/remote.py @@ -68,9 +68,7 @@ def __init__(self, ctx, mechanism): self.mechanism = BullsharkMechanism(self.settings) try: - ctx.connect_kwargs.pkey = RSAKey.from_private_key_file( - self.manager.settings.key_path - ) + ctx.connect_kwargs.pkey = RSAKey.from_private_key_file(self.manager.settings.key_path) self.connect = ctx.connect_kwargs except (IOError, PasswordRequiredException, SSHException) as e: raise BenchError("Failed to load SSH key", e) @@ -152,9 +150,7 @@ def _config(self, isGeoremote, hosts, node_parameters, bench_parameters=None): # Cleanup node configuration files on hosts for i, host in enumerate(hosts): cmd = CommandMaker.clean_node_config(i) - c = Connection( - host, user=self.settings.key_name, connect_kwargs=self.connect - ) + c = Connection(host, user=self.settings.key_name, connect_kwargs=self.connect) c.run(cmd, shell=True) # Create persistent peers @@ -185,22 +181,16 @@ def _config(self, isGeoremote, hosts, node_parameters, bench_parameters=None): # Upload configuration files. progress = progress_bar(hosts, prefix="Uploading config files:") for i, host in enumerate(hosts): - cmd = [ - f"scp -i {self.settings.key_path} -r ~/geodec/mytestnet/node{i} ubuntu@{host}:~/" - ] # NOTE Path of the node config files + cmd = [f"scp -i {self.settings.key_path} -r ~/geodec/mytestnet/node{i} ubuntu@{host}:~/"] # NOTE Path of the node config files subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL) else: # Recompile the latest code. cmd = CommandMaker.compile().split() - subprocess.run( - cmd, check=True, cwd=PathMaker.node_crate_path(self.settings.repo_name) - ) + subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path(self.settings.repo_name)) # Create alias for the client and nodes binary. - cmd = CommandMaker.alias_binaries( - PathMaker.binary_path(self.settings.repo_name), self.mechanism.name - ) + cmd = CommandMaker.alias_binaries(PathMaker.binary_path(self.settings.repo_name), self.mechanism.name) subprocess.run([cmd], shell=True) # Generate configuration files. @@ -214,18 +204,14 @@ def _config(self, isGeoremote, hosts, node_parameters, bench_parameters=None): names = [x.name for x in keys] if self.mechanism.name == "hotstuff": - consensus_addr = [ - f'{x}:{self.settings.ports["consensus"]}' for x in hosts - ] + consensus_addr = [f'{x}:{self.settings.ports["consensus"]}' for x in hosts] front_addr = [f'{x}:{self.settings.ports["front"]}' for x in hosts] mempool_addr = [f'{x}:{self.settings.ports["mempool"]}' for x in hosts] committee = Committee(names, consensus_addr, front_addr, mempool_addr) elif self.mechanism.name == "bullshark": if bench_parameters.collocate: workers = bench_parameters.workers - addresses = OrderedDict( - (x, [y] * (workers + 1)) for x, y in zip(names, hosts) - ) + addresses = OrderedDict((x, [y] * (workers + 1)) for x, y in zip(names, hosts)) else: addresses = OrderedDict((x, y) for x, y in zip(names, hosts)) committee = BullsharkCommittee(addresses, self.settings.ports["base"]) @@ -243,18 +229,14 @@ def _config(self, isGeoremote, hosts, node_parameters, bench_parameters=None): # NOTE Upload configuration files. progress = progress_bar(hosts, prefix="Uploading config files:") for i, host in enumerate(progress): - c = Connection( - host, user=self.settings.key_name, connect_kwargs=self.connect - ) + c = Connection(host, user=self.settings.key_name, connect_kwargs=self.connect) c.put(PathMaker.committee_file(), ".") c.put(PathMaker.key_file(i), ".") c.put(PathMaker.parameters_file(), ".") return committee - def _run_single( - self, hosts, rate, bench_parameters, node_parameters, debug=False, committee=[] - ): + def _run_single(self, hosts, rate, bench_parameters, node_parameters, debug=False, committee=[]): Print.info("Booting testbed...") # Kill any potentially unfinished run and delete logs. @@ -311,9 +293,7 @@ def _run_single( addresses = [f'{x}:{self.settings.ports["front"]}' for x in hosts] # rate_share = ceil(rate / committee.size()) # TODO Take faults into account. rate_share = ceil(rate / len(hosts)) - duration = ( - bench_parameters.duration - ) # Duration for which the client should run + duration = bench_parameters.duration # Duration for which the client should run client_logs = [PathMaker.client_log_file(i) for i in range(len(hosts))] for host, addr, log_file in zip(hosts, addresses, client_logs): cmd = CommandMaker.run_client( @@ -406,9 +386,7 @@ def _logs(self, hosts, faults, committee=[]): # , servers, run_id): progress = progress_bar(hosts, prefix="Downloading logs:") if self.mechanism.name == "bullshark": workers_addresses = committee.workers_addresses(faults) - progress = progress_bar( - workers_addresses, prefix="Downloading workers logs:" - ) + progress = progress_bar(workers_addresses, prefix="Downloading workers logs:") for i, addresses in enumerate(progress): for id, address in addresses: host = BullsharkCommittee.ip(address) @@ -423,20 +401,14 @@ def _logs(self, hosts, faults, committee=[]): # , servers, run_id): ) primary_addresses = committee.primary_addresses(faults) - progress = progress_bar( - primary_addresses, prefix="Downloading primaries logs:" - ) + progress = progress_bar(primary_addresses, prefix="Downloading primaries logs:") for i, address in enumerate(progress): host = BullsharkCommittee.ip(address) c = Connection(host, user="ubuntu", connect_kwargs=self.connect) - c.get( - PathMaker.primary_log_file(i), local=PathMaker.primary_log_file(i) - ) + c.get(PathMaker.primary_log_file(i), local=PathMaker.primary_log_file(i)) else: for i, host in enumerate(progress): - c = Connection( - host, user=self.settings.key_name, connect_kwargs=self.connect - ) + c = Connection(host, user=self.settings.key_name, connect_kwargs=self.connect) c.get(PathMaker.node_log_file(i), local=PathMaker.node_log_file(i)) c.get(PathMaker.client_log_file(i), local=PathMaker.client_log_file(i)) if self.mechanism.name == "cometbft": @@ -452,9 +424,7 @@ def _logs(self, hosts, faults, committee=[]): # , servers, run_id): logParser.log_parser(self.mechanism.name, PathMaker.logs_path(), faults=faults) return logParser - def run( - self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=False - ): + def run(self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=False): assert isinstance(debug, bool) Print.heading(f"Starting {self.mechanism.name} remote benchmark") @@ -483,25 +453,13 @@ def run( if isGeoRemote: geo_input = GeoDec.getGeoInput(self.settings.geo_input) - selected_servers = GeoDec.getAllServers( - geo_input, self.settings.servers_file, self.settings.ip_file - ) - pingDelays = GeoDec.getPingDelay( - geo_input, self.settings.ping_grouped_file, self.settings.pings_file - ) + selected_servers = GeoDec.getAllServers(geo_input, self.settings.servers_file, self.settings.ip_file) + pingDelays = GeoDec.getPingDelay(geo_input, self.settings.ping_grouped_file, self.settings.pings_file) Print.heading("\nSelected servers:") - print( - selected_servers[ - ["ip", "id", "name", "latitude", "longitude"] - ].to_string(index=False) - ) + print(selected_servers[["ip", "id", "name", "latitude", "longitude"]].to_string(index=False)) Print.heading("\nPing Delays:") - print( - pingDelays[["source", "destination", "avg", "mdev"]].to_string( - index=False - ) - ) + print(pingDelays[["source", "destination", "avg", "mdev"]].to_string(index=False)) if len(pingDelays) != len(selected_servers) * (len(selected_servers) - 1): print("ERROR: Ping delays not available for all servers") @@ -516,9 +474,7 @@ def run( try: latencySetter.configDelay(selected_hosts) - latencySetter.addDelays( - selected_servers, pingDelays, self.settings.interface - ) + latencySetter.addDelays(selected_servers, pingDelays, self.settings.interface) except (subprocess.SubprocessError, GroupException) as e: e = FabricError(e) if isinstance(e, GroupException) else e Print.error(BenchError("Failed to initalize delays", e)) @@ -531,9 +487,7 @@ def run( # Upload all configuration files. try: - committee = self._config( - isGeoRemote, hosts, node_parameters, bench_parameters - ) + committee = self._config(isGeoRemote, hosts, node_parameters, bench_parameters) except (subprocess.SubprocessError, GroupException) as e: e = FabricError(e) if isinstance(e, GroupException) else e Print.error(BenchError("Failed to configure nodes", e)) @@ -553,9 +507,7 @@ def run( # Run the benchmark. for i in range(bench_parameters.runs): run_id = GeoLogParser.get_new_run_id() - Print.heading( - f"Run {i+1}/{bench_parameters.runs} with run_id {run_id}" - ) + Print.heading(f"Run {i+1}/{bench_parameters.runs} with run_id {run_id}") try: self._run_single( From a30102fd6fbfd096bdc4b236d4df26e085547690 Mon Sep 17 00:00:00 2001 From: Naman Garg <0708ng@gmail.com> Date: Sun, 14 Jul 2024 17:53:40 +0530 Subject: [PATCH 04/11] feat: add result csv --- benchmark/geo_logs.py | 25 -------- benchmark/logs.py | 132 ++++++++++++++++++++++++++++++++++++++---- benchmark/remote.py | 1 - 3 files changed, 120 insertions(+), 38 deletions(-) diff --git a/benchmark/geo_logs.py b/benchmark/geo_logs.py index 2279f2c..0140e72 100644 --- a/benchmark/geo_logs.py +++ b/benchmark/geo_logs.py @@ -71,28 +71,3 @@ def _calculate_liveliness(data, total_block_commits): total_props = data["proposals"].sum() data["liveliness_woprops"] = ((data["votes"] + data["proposals"]) / total_props) * 100 return data - - @staticmethod - def get_new_run_id(): - data = pd.read_csv("/home/ubuntu/results/metrics.csv") - return data["run_id"].max() + 1 - - @staticmethod - def aggregate_runs(run_id_array): - data = pd.read_csv("/home/ubuntu/results/geo-dec-metrics.csv") - - data = data.loc[data["run_id"].isin(run_id_array)] - by_name = data.groupby(["name"]) - - # for name, liveliness in by_name: - # print(f"entries for {name!r}") - # print("------------------------") - # print(liveliness.head(3), end="\n\n") - - liveliness_mean = by_name["liveliness"].mean(numeric_only=True).reset_index() - liveliness_mean.rename(columns={"liveliness": "liveliness_avg"}, inplace=True) - - data_first = data.loc[data["run_id"] == run_id_array[0]] - result = pd.merge(data_first, liveliness_mean, on="name") - result["runs"] = [len(run_id_array)] * len(result) - return result diff --git a/benchmark/logs.py b/benchmark/logs.py index dbf83f3..1bb771a 100644 --- a/benchmark/logs.py +++ b/benchmark/logs.py @@ -1,11 +1,19 @@ +import csv import glob +import re from os.path import join + +import pandas as pd + +from benchmark.mechanisms.bullshark import BullsharkLogParser from benchmark.mechanisms.cometbft import CometBftLogParser from benchmark.mechanisms.hotstuff import HotStuffLogParser -from benchmark.mechanisms.bullshark import BullsharkLogParser + class ParseError(Exception): pass + + class LogParser: def __init__(self): self.result_str = "" @@ -15,34 +23,134 @@ def process(cls, directory, faults): assert isinstance(directory, str) clients = [] - for filename in sorted(glob.glob(join(directory, 'client-*.log'))): - with open(filename, 'r') as f: + for filename in sorted(glob.glob(join(directory, "client-*.log"))): + with open(filename, "r") as f: clients.append(f.read()) nodes = [] - for filename in sorted(glob.glob(join(directory, 'node-*.log'))): - with open(filename, 'r') as f: + for filename in sorted(glob.glob(join(directory, "node-*.log"))): + with open(filename, "r") as f: nodes.append(f.read()) latency = [] - for filename in sorted(glob.glob(join(directory, 'latency-*.log'))): - with open(filename, 'r') as f: + for filename in sorted(glob.glob(join(directory, "latency-*.log"))): + with open(filename, "r") as f: latency.append(f.read()) return cls(clients, nodes, latency, faults) - + def result(self): return self.result_str def print(self, filename): assert isinstance(filename, str) - with open(filename, 'a') as f: + + print(self.result_str) + + with open(filename, "a") as f: f.write(self.result_str) + result_json = self.parse_results() + write_results_to_csv(result_json, "results.csv") + def log_parser(self, mechanism_name, directory, faults=0): if mechanism_name == "hotstuff": - result = HotStuffLogParser.process(directory, faults).result_str + result = HotStuffLogParser.process(directory, faults).result_str elif mechanism_name == "cometbft": result = CometBftLogParser.process(directory, faults).result_str elif mechanism_name == "bullshark": result = BullsharkLogParser.process(directory, faults).result_str - - self.result_str = result \ No newline at end of file + + self.result_str = result + + @staticmethod + def get_new_run_id(): + data = pd.read_csv("/home/ubuntu/geodec/results/metrics.csv") + return data["run_id"].max() + 1 + + @staticmethod + def aggregate_runs(run_id_array): + data = pd.read_csv("/home/ubuntu/geodec/results/metrics.csv") + + data = data.loc[data["run_id"].isin(run_id_array)] + by_name = data.groupby(["name"]) + + # for name, liveliness in by_name: + # print(f"entries for {name!r}") + # print("------------------------") + # print(liveliness.head(3), end="\n\n") + + liveliness_mean = by_name["liveliness"].mean(numeric_only=True).reset_index() + liveliness_mean.rename(columns={"liveliness": "liveliness_avg"}, inplace=True) + + data_first = data.loc[data["run_id"] == run_id_array[0]] + result = pd.merge(data_first, liveliness_mean, on="name") + result["runs"] = [len(run_id_array)] * len(result) + return result + + def parse_results(self): + results = {} + + lines = self.result_str.split("\n") + + results["run_id"] = self.get_new_run_id() + 1 + results["name"] = "" + + mechanism_match = re.match(r"^\s*(\w+)\s+SUMMARY:", lines[0]) + if mechanism_match: + results["mechanism"] = mechanism_match.group(1) + + # Parsing the CONFIG section + for line in lines: + if line.startswith(" Faults:"): + results["faults"] = int(line.split(":")[1].strip().split(" ")[0]) + elif line.startswith(" Input rate:"): + results["input_rate"] = int(line.split(":")[1].strip().split(" ")[0]) + elif line.startswith(" Committee size:"): + results["committee_size"] = int(line.split(":")[1].strip().split(" ")[0]) + elif line.startswith(" Transaction size:"): + results["transaction_size"] = int(line.split(":")[1].strip().split(" ")[0]) + elif line.startswith(" Execution time:"): + results["execution_time"] = int(line.split(":")[1].strip().split(" ")[0]) + elif line.startswith(" Mempool batch size:"): + results["batch_size"] = int(line.split(":")[1].strip().split(" ")[0]) + + # Parsing the RESULTS section + for line in lines: + if line.startswith(" Consensus TPS:"): + results["consensus_tps"] = int(line.split(":")[1].strip().split(" ")[0]) + elif line.startswith(" Consensus BPS:"): + results["consensus_bps"] = int(line.split(":")[1].strip().split(" ")[0].replace(",", "")) + elif line.startswith(" Consensus latency:"): + results["consensus_latency"] = int(line.split(":")[1].strip().split(" ")[0]) + elif line.startswith(" End-to-end TPS:"): + results["end_to_end_tps"] = int(line.split(":")[1].strip().split(" ")[0]) + elif line.startswith(" End-to-end BPS:"): + results["end_to_end_bps"] = int(line.split(":")[1].strip().split(" ")[0].replace(",", "")) + elif line.startswith(" End-to-end latency:"): + results["end_to_end_latency"] = int(line.split(":")[1].strip().split(" ")[0]) + + return results + + +def write_results_to_csv(results, csv_filename): + fieldnames = [ + "run_id", + "name", + "faults", + "input_rate", + "committee_size", + "transaction_size", + "execution_time", + "batch_size", + "consensus_tps", + "consensus_bps", + "consensus_latency", + "end_to_end_tps", + "end_to_end_bps", + "end_to_end_latency", + ] + + with open(csv_filename, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + + writer.writeheader() + writer.writerow(results) diff --git a/benchmark/remote.py b/benchmark/remote.py index 5f87c28..39a9a50 100644 --- a/benchmark/remote.py +++ b/benchmark/remote.py @@ -520,7 +520,6 @@ def run(self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=Fa ) logger = self._logs(hosts, faults, committee_copy) - print(logger.result()) logger.print( PathMaker.result_file( self.mechanism.name, From 589dfba6360ff4bc0157872b33c4463992cc8f6c Mon Sep 17 00:00:00 2001 From: Naman Garg <0708ng@gmail.com> Date: Sun, 14 Jul 2024 18:01:35 +0530 Subject: [PATCH 05/11] feat: multiple runs aggregation --- benchmark/logs.py | 76 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/benchmark/logs.py b/benchmark/logs.py index 1bb771a..54f07b3 100644 --- a/benchmark/logs.py +++ b/benchmark/logs.py @@ -5,6 +5,7 @@ import pandas as pd +# Import parsers for specific mechanisms from benchmark.mechanisms.bullshark import BullsharkLogParser from benchmark.mechanisms.cometbft import CometBftLogParser from benchmark.mechanisms.hotstuff import HotStuffLogParser @@ -20,17 +21,21 @@ def __init__(self): @classmethod def process(cls, directory, faults): - assert isinstance(directory, str) + assert isinstance(directory, str), "Directory path must be a string" - clients = [] + clients, nodes, latency = [], [], [] + + # Read client logs for filename in sorted(glob.glob(join(directory, "client-*.log"))): with open(filename, "r") as f: clients.append(f.read()) - nodes = [] + + # Read node logs for filename in sorted(glob.glob(join(directory, "node-*.log"))): with open(filename, "r") as f: nodes.append(f.read()) - latency = [] + + # Read latency logs for filename in sorted(glob.glob(join(directory, "latency-*.log"))): with open(filename, "r") as f: latency.append(f.read()) @@ -41,7 +46,7 @@ def result(self): return self.result_str def print(self, filename): - assert isinstance(filename, str) + assert isinstance(filename, str), "Filename must be a string" print(self.result_str) @@ -58,42 +63,59 @@ def log_parser(self, mechanism_name, directory, faults=0): result = CometBftLogParser.process(directory, faults).result_str elif mechanism_name == "bullshark": result = BullsharkLogParser.process(directory, faults).result_str + else: + raise ParseError(f"Unknown mechanism: {mechanism_name}") self.result_str = result @staticmethod def get_new_run_id(): - data = pd.read_csv("/home/ubuntu/geodec/results/metrics.csv") - return data["run_id"].max() + 1 + try: + data = pd.read_csv("/home/ubuntu/geodec/results/metrics.csv") + return data["run_id"].max() + 1 + except FileNotFoundError: + return 1 # Return 1 if the file doesn't exist @staticmethod def aggregate_runs(run_id_array): data = pd.read_csv("/home/ubuntu/geodec/results/metrics.csv") + # Filter data to include only the specified run IDs data = data.loc[data["run_id"].isin(run_id_array)] - by_name = data.groupby(["name"]) - # for name, liveliness in by_name: - # print(f"entries for {name!r}") - # print("------------------------") - # print(liveliness.head(3), end="\n\n") + # Define the fields to average + fields_to_avg = [ + "consensus_tps", + "consensus_bps", + "consensus_latency", + "end_to_end_tps", + "end_to_end_bps", + "end_to_end_latency", + ] + + # Group by 'name' and calculate the average for the specified fields + averages = data.groupby("name")[fields_to_avg].mean().reset_index() - liveliness_mean = by_name["liveliness"].mean(numeric_only=True).reset_index() - liveliness_mean.rename(columns={"liveliness": "liveliness_avg"}, inplace=True) + # Replace the original data with the averages + for field in fields_to_avg: + data[field] = data["name"].map(averages.set_index("name")[field]) - data_first = data.loc[data["run_id"] == run_id_array[0]] - result = pd.merge(data_first, liveliness_mean, on="name") - result["runs"] = [len(run_id_array)] * len(result) - return result + # Remove duplicates, keeping only the first occurrence + data = data.drop_duplicates(subset="name").reset_index(drop=True) + + # Add a column to indicate the number of runs aggregated + data["runs"] = len(run_id_array) + + return data def parse_results(self): results = {} - lines = self.result_str.split("\n") - results["run_id"] = self.get_new_run_id() + 1 + results["run_id"] = self.get_new_run_id() results["name"] = "" + # Extract mechanism name from the summary header mechanism_match = re.match(r"^\s*(\w+)\s+SUMMARY:", lines[0]) if mechanism_match: results["mechanism"] = mechanism_match.group(1) @@ -149,8 +171,18 @@ def write_results_to_csv(results, csv_filename): "end_to_end_latency", ] - with open(csv_filename, "w", newline="") as csvfile: + # Append to CSV if it exists, otherwise create it + file_exists = False + try: + with open(csv_filename, "r"): + file_exists = True + except FileNotFoundError: + pass + + with open(csv_filename, "a" if file_exists else "w", newline="") as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() + if not file_exists: + writer.writeheader() # Write header only if file doesn't exist + writer.writerow(results) From 0a6cc571b9452ff632a77d80602f237395cca51f Mon Sep 17 00:00:00 2001 From: Naman Garg <0708ng@gmail.com> Date: Sun, 14 Jul 2024 18:08:44 +0530 Subject: [PATCH 06/11] fix: aggregation --- benchmark/logs.py | 65 ++++++++++++++++++++++++++++----------------- benchmark/remote.py | 15 ++++------- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/benchmark/logs.py b/benchmark/logs.py index 54f07b3..9cd143c 100644 --- a/benchmark/logs.py +++ b/benchmark/logs.py @@ -54,7 +54,7 @@ def print(self, filename): f.write(self.result_str) result_json = self.parse_results() - write_results_to_csv(result_json, "results.csv") + write_results_to_csv(result_json, "/home/ubuntu/geodec/results/metrics.csv") def log_parser(self, mechanism_name, directory, faults=0): if mechanism_name == "hotstuff": @@ -78,35 +78,52 @@ def get_new_run_id(): @staticmethod def aggregate_runs(run_id_array): - data = pd.read_csv("/home/ubuntu/geodec/results/metrics.csv") - - # Filter data to include only the specified run IDs - data = data.loc[data["run_id"].isin(run_id_array)] - - # Define the fields to average - fields_to_avg = [ - "consensus_tps", - "consensus_bps", - "consensus_latency", - "end_to_end_tps", - "end_to_end_bps", - "end_to_end_latency", + csv_file = "/home/ubuntu/geodec/results/metrics.csv" + data = pd.read_csv(csv_file) + + # Filter data for the given run IDs + data_to_aggregate = data.loc[data["run_id"].isin(run_id_array)] + + # Compute the mean for the specified fields + aggregated_data = data_to_aggregate.mean(numeric_only=True).reset_index() + aggregated_data = aggregated_data.loc[ + aggregated_data["index"].isin( + [ + "consensus_tps", + "consensus_bps", + "consensus_latency", + "end_to_end_tps", + "end_to_end_bps", + "end_to_end_latency", + ] + ) ] - # Group by 'name' and calculate the average for the specified fields - averages = data.groupby("name")[fields_to_avg].mean().reset_index() + # Convert the result to a dictionary for easy updating + aggregated_results = aggregated_data.set_index("index").T.to_dict("records")[0] - # Replace the original data with the averages - for field in fields_to_avg: - data[field] = data["name"].map(averages.set_index("name")[field]) + # Add non-aggregated fields + run_id = min(run_id_array) + aggregated_results["run_id"] = run_id + aggregated_results["name"] = data_to_aggregate.iloc[0]["name"] + aggregated_results["faults"] = data_to_aggregate.iloc[0]["faults"] + aggregated_results["input_rate"] = data_to_aggregate.iloc[0]["input_rate"] + aggregated_results["committee_size"] = data_to_aggregate.iloc[0]["committee_size"] + aggregated_results["transaction_size"] = data_to_aggregate.iloc[0]["transaction_size"] + aggregated_results["execution_time"] = data_to_aggregate.iloc[0]["execution_time"] + aggregated_results["batch_size"] = data_to_aggregate.iloc[0]["batch_size"] - # Remove duplicates, keeping only the first occurrence - data = data.drop_duplicates(subset="name").reset_index(drop=True) + # Remove the original data for the specified run IDs + data = data.loc[~data["run_id"].isin(run_id_array)] - # Add a column to indicate the number of runs aggregated - data["runs"] = len(run_id_array) + # Append the new aggregated data + new_data = pd.DataFrame([aggregated_results]) + data = pd.concat([data, new_data], ignore_index=True) - return data + # Save the updated data back to the CSV file + data.to_csv(csv_file, index=False) + + return aggregated_results def parse_results(self): results = {} diff --git a/benchmark/remote.py b/benchmark/remote.py index 39a9a50..cd7d859 100644 --- a/benchmark/remote.py +++ b/benchmark/remote.py @@ -20,7 +20,6 @@ Key, NodeParameters, ) -from benchmark.geo_logs import GeoLogParser from benchmark.geodec import GeoDec from benchmark.instance import InstanceManager from benchmark.latency_setter import LatencySetter @@ -181,7 +180,9 @@ def _config(self, isGeoremote, hosts, node_parameters, bench_parameters=None): # Upload configuration files. progress = progress_bar(hosts, prefix="Uploading config files:") for i, host in enumerate(hosts): - cmd = [f"scp -i {self.settings.key_path} -r ~/geodec/mytestnet/node{i} ubuntu@{host}:~/"] # NOTE Path of the node config files + cmd = [ + f"scp -i {self.settings.key_path} -r ~/geodec/mytestnet/node{i} ubuntu@{host}:~/" + ] # NOTE Path of the node config files subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL) else: @@ -506,7 +507,7 @@ def run(self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=Fa # Run the benchmark. for i in range(bench_parameters.runs): - run_id = GeoLogParser.get_new_run_id() + run_id = LogParser.get_new_run_id() Print.heading(f"Run {i+1}/{bench_parameters.runs} with run_id {run_id}") try: @@ -542,14 +543,8 @@ def run(self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=Fa Print.error(BenchError("Benchmark failed", e)) continue - aggregated_results = GeoLogParser.aggregate_runs(run_id_array) + aggregated_results = LogParser.aggregate_runs(run_id_array) print(aggregated_results) - aggregated_results.to_csv( - "/home/ubuntu/results/64node-fixed-mean-geo-dec-metrics.csv", - mode="a", - index=False, - header=False, - ) if isGeoRemote: # Delete delay parameters. From 36361573173fa76fbb96ea5c439fa039d7bd38bd Mon Sep 17 00:00:00 2001 From: Naman Garg <0708ng@gmail.com> Date: Sun, 14 Jul 2024 13:29:07 +0000 Subject: [PATCH 07/11] fix: log parsing --- .gitignore | 2 +- benchmark/logs.py | 71 ++++++++++++++++++++++++++++++++++--------- benchmark/remote.py | 3 +- fab-params.json | 8 ++--- results/metrics.csv | 8 +++++ rundata/geo_input.csv | 12 +++----- 6 files changed, 75 insertions(+), 29 deletions(-) create mode 100644 results/metrics.csv diff --git a/.gitignore b/.gitignore index 94acc89..7ac4af0 100644 --- a/.gitignore +++ b/.gitignore @@ -141,7 +141,7 @@ node client mytestnet/ -persistent_peer.txt +*.txt frontend/node_modules diff --git a/benchmark/logs.py b/benchmark/logs.py index 9cd143c..7a45db8 100644 --- a/benchmark/logs.py +++ b/benchmark/logs.py @@ -18,6 +18,7 @@ class ParseError(Exception): class LogParser: def __init__(self): self.result_str = "" + self.name = "" @classmethod def process(cls, directory, faults): @@ -57,6 +58,7 @@ def print(self, filename): write_results_to_csv(result_json, "/home/ubuntu/geodec/results/metrics.csv") def log_parser(self, mechanism_name, directory, faults=0): + self.name = mechanism_name if mechanism_name == "hotstuff": result = HotStuffLogParser.process(directory, faults).result_str elif mechanism_name == "cometbft": @@ -103,8 +105,8 @@ def aggregate_runs(run_id_array): aggregated_results = aggregated_data.set_index("index").T.to_dict("records")[0] # Add non-aggregated fields - run_id = min(run_id_array) - aggregated_results["run_id"] = run_id + run_id = max(run_id_array) + aggregated_results["run_id"] = run_id + 1 aggregated_results["name"] = data_to_aggregate.iloc[0]["name"] aggregated_results["faults"] = data_to_aggregate.iloc[0]["faults"] aggregated_results["input_rate"] = data_to_aggregate.iloc[0]["input_rate"] @@ -112,9 +114,10 @@ def aggregate_runs(run_id_array): aggregated_results["transaction_size"] = data_to_aggregate.iloc[0]["transaction_size"] aggregated_results["execution_time"] = data_to_aggregate.iloc[0]["execution_time"] aggregated_results["batch_size"] = data_to_aggregate.iloc[0]["batch_size"] + aggregated_results['runs'] = len(run_id_array) - # Remove the original data for the specified run IDs - data = data.loc[~data["run_id"].isin(run_id_array)] + # # Remove the original data for the specified run IDs + # data = data.loc[~data["run_id"].isin(run_id_array)] # Append the new aggregated data new_data = pd.DataFrame([aggregated_results]) @@ -123,22 +126,16 @@ def aggregate_runs(run_id_array): # Save the updated data back to the CSV file data.to_csv(csv_file, index=False) - return aggregated_results - def parse_results(self): results = {} lines = self.result_str.split("\n") results["run_id"] = self.get_new_run_id() - results["name"] = "" - - # Extract mechanism name from the summary header - mechanism_match = re.match(r"^\s*(\w+)\s+SUMMARY:", lines[0]) - if mechanism_match: - results["mechanism"] = mechanism_match.group(1) + results["name"] = self.name # Parsing the CONFIG section for line in lines: + line = line.replace(",", "") if line.startswith(" Faults:"): results["faults"] = int(line.split(":")[1].strip().split(" ")[0]) elif line.startswith(" Input rate:"): @@ -154,16 +151,17 @@ def parse_results(self): # Parsing the RESULTS section for line in lines: + line = line.replace(",", "") if line.startswith(" Consensus TPS:"): results["consensus_tps"] = int(line.split(":")[1].strip().split(" ")[0]) elif line.startswith(" Consensus BPS:"): - results["consensus_bps"] = int(line.split(":")[1].strip().split(" ")[0].replace(",", "")) + results["consensus_bps"] = int(line.split(":")[1].strip().split(" ")[0]) elif line.startswith(" Consensus latency:"): results["consensus_latency"] = int(line.split(":")[1].strip().split(" ")[0]) elif line.startswith(" End-to-end TPS:"): results["end_to_end_tps"] = int(line.split(":")[1].strip().split(" ")[0]) elif line.startswith(" End-to-end BPS:"): - results["end_to_end_bps"] = int(line.split(":")[1].strip().split(" ")[0].replace(",", "")) + results["end_to_end_bps"] = int(line.split(":")[1].strip().split(" ")[0]) elif line.startswith(" End-to-end latency:"): results["end_to_end_latency"] = int(line.split(":")[1].strip().split(" ")[0]) @@ -203,3 +201,48 @@ def write_results_to_csv(results, csv_filename): writer.writeheader() # Write header only if file doesn't exist writer.writerow(results) + + +# Test function +def test_log_parser(): + log_parser = LogParser() + + # Read from a test result.txt file + with open("result.txt", "r") as f: + log_parser.result_str = f.read() + + # Parse results and write to CSV + results = log_parser.parse_results() + write_results_to_csv(results, "results.csv") + + # Aggregate and update CSV + run_id_array = [results["run_id"]] # Using the run_id from the parsed results + aggregated_data = log_parser.aggregate_runs(run_id_array) + + print("Aggregated Data:") + print(aggregated_data) + + +# Example content for result.txt +example_result_txt = """ +COMETBFT SUMMARY: +Date and Time: 2024-07-04 11:25:28 + + Faults: 0 nodes + Committee size: 4 nodes + Input rate: 1,000 tx/s + Transaction size: 256 B + Execution time: 101 s + + Consensus timeout delay: 5,000 ms + Consensus sync retry delay: 5,000 ms + Mempool GC depth: 50 rounds + Mempool sync retry delay: 5,000 ms + Mempool sync retry nodes: 3 nodes + Mempool batch size: 204,800 B + Mempool max batch delay: 100 ms + + Consensus TPS: 960 tx/s + Consensus BPS: 245,817 B/s + Consensus latency: 3 + """ diff --git a/benchmark/remote.py b/benchmark/remote.py index cd7d859..da33fa6 100644 --- a/benchmark/remote.py +++ b/benchmark/remote.py @@ -543,8 +543,7 @@ def run(self, bench_parameters_dict, node_parameters_dict, isGeoRemote, debug=Fa Print.error(BenchError("Benchmark failed", e)) continue - aggregated_results = LogParser.aggregate_runs(run_id_array) - print(aggregated_results) + LogParser.aggregate_runs(run_id_array) if isGeoRemote: # Delete delay parameters. diff --git a/fab-params.json b/fab-params.json index abcc294..95c98f8 100644 --- a/fab-params.json +++ b/fab-params.json @@ -3,11 +3,11 @@ "hotstuff": { "bench_params": { "faults": 0, - "nodes": [8], + "nodes": [4], "rate": [100000], "tx_size": 256, - "duration": 100, - "runs": 1 + "duration": 60, + "runs": 3 }, "node_params": { "consensus": { @@ -52,7 +52,7 @@ "nodes": [4], "workers": 1, "collocate": true, - "rate": [10000], + "rate": [100000], "tx_size": 256, "duration": 100, "runs": 1 diff --git a/results/metrics.csv b/results/metrics.csv new file mode 100644 index 0000000..a60e796 --- /dev/null +++ b/results/metrics.csv @@ -0,0 +1,8 @@ +run_id,name,faults,input_rate,committee_size,transaction_size,execution_time,batch_size,consensus_tps,consensus_bps,consensus_latency,end_to_end_tps,end_to_end_bps,end_to_end_latency,runs +1,,0.0,100000.0,4.0,256.0,60.0,204800.0,100033.0,25608515.5,13.0,99993.5,25598293.5,110.5, +2,,0.0,100000.0,4.0,256.0,56.0,204800.0,100026.33333333331,25606755.0,55.333333333333336,100005.33333333331,25601317.0,266.3333333333333, +3,hotstuff,0.0,100000.0,4.0,256.0,61.0,204800.0,100234.0,25659932.0,5.666666666666667,100003.0,25600719.666666668,33.0, +4,hotstuff,0.0,100000.0,4.0,256.0,59.0,204800.0,100007.0,25601867.0,16.0,99934.0,25583121.0,173.0, +5,hotstuff,0.0,100000.0,4.0,256.0,60.0,204800.0,99965.0,25591072.0,38.0,99944.0,25585548.0,264.0, +6,hotstuff,0.0,100000.0,4.0,256.0,60.0,204800.0,99980.0,25594813.0,18.0,99955.0,25588454.0,242.0, +7,hotstuff,0.0,100000.0,4.0,256.0,59.0,204800.0,99984.0,25595917.333333332,24.0,99944.33333333333,25585707.666666668,226.33333333333334,3.0 diff --git a/rundata/geo_input.csv b/rundata/geo_input.csv index 0c33a17..9202f12 100644 --- a/rundata/geo_input.csv +++ b/rundata/geo_input.csv @@ -1,9 +1,5 @@ id,name,title,location,state,country,state_abbv,continent,latitude,longitude,count,stake -151,Honolulu,Honolulu,Honolulu,Hawaii,United States,HI,1,21.3,-157.8167,1,1 -103,Edmonton,Edmonton,St Albert,Alberta,Canada,AB,1,53.6351,-113.6216,1,1 -167,Montevideo,Montevideo,Pando,Canelones,Uruguay,CA,2,-34.7249,-55.9477,1,1 -168,SanJuan,San Juan,San Juan,,Puerto Rico,,1,18.45,-66.0667,1,1 -118,Kampala,Kampala,Kampala,Kampala,Uganda,,5,0.3136,32.5811,1,1 -10,Stockholm,Stockholm,Stockholm,Stockholm,Sweden,,3,59.32,18.09,1,1 -21,Brisbane,Brisbane,Brisbane,Queensland,Australia,QLD,4,-27.4667,153.0333,1,1 -261,Zhangjiakou,Zhangjiakou,Zhangjiakou,Hebei,China,HE,3,40.7675,114.8863,1,1 \ No newline at end of file +90,StLouis,St Louis,St Louis,Missouri,United States,MO,1,38.63,-90.2,1,1 +125,SaoPaulo,Sao Paulo,Sao Paulo,Sao Paulo,Brazil,SP,2,-23.55,-46.6333,1,1 +32,Warsaw,Warsaw,Warsaw,Mazovia,Poland,MZ,3,52.23,21.0108,1,1 +169,Perth,Perth,Perth,Western Australia,Australia,WA,4,-31.9522,115.8589,1,10 \ No newline at end of file From b43114e1bee6825efd62ae7677ff7302a191d30b Mon Sep 17 00:00:00 2001 From: Naman Garg <0708ng@gmail.com> Date: Sun, 14 Jul 2024 19:16:20 +0530 Subject: [PATCH 08/11] update geo run --- benchmark/geo_runs.py | 244 ++++++++++++++++++++++++++---------------- fab-params.json | 14 +-- 2 files changed, 160 insertions(+), 98 deletions(-) diff --git a/benchmark/geo_runs.py b/benchmark/geo_runs.py index bfc8ed3..5fa5012 100644 --- a/benchmark/geo_runs.py +++ b/benchmark/geo_runs.py @@ -1,23 +1,26 @@ #!/usr/bin/python3 import datetime +import json +import math import os -import pandas as pd import random -from re import sub import subprocess import sys -import math - +from re import sub from time import sleep + +import pandas as pd + from geodec import GeoDec -SERVERS_FILE = '/home/ubuntu/data/servers-2020-07-19.csv' -MARKED_SERVERS_FILE = '/home/ubuntu/data/servers-2020-07-19-us-europe-filter-2.csv' -COMMITTEE_SIZE = 64 #16 +SERVERS_FILE = "/home/ubuntu/data/servers-2020-07-19.csv" +MARKED_SERVERS_FILE = "/home/ubuntu/data/servers-2020-07-19-us-europe-filter-2.csv" +COMMITTEE_SIZE = 64 # 16 + def change_config(config, rate, batch_size, message_size): - with open(config, 'r') as f: + with open(config, "r") as f: lines = f.readlines() flag = False @@ -33,11 +36,12 @@ def change_config(config, rate, batch_size, message_size): lines[i] = f" 'tx_size': {message_size},\n" elif "'batch_size':" in lines[i]: lines[i] = f" 'batch_size': {batch_size * message_size},\n" - with open(config, 'w') as f: + with open(config, "w") as f: f.writelines(lines) + def change_location_input(config, geo_input): - with open(config, 'r') as f: + with open(config, "r") as f: lines = f.readlines() flag = False for i in range(len(lines)): @@ -46,14 +50,16 @@ def change_location_input(config, geo_input): if flag: if lines[i].startswith(" geoInput"): lines[i] = f" geoInput = {geo_input}\n" - - with open(config, 'w') as f: + + with open(config, "w") as f: f.writelines(lines) + def get_server_locations(self): - servers_locs = pd.read_csv(SERVERS_FILE)['id'].values.tolist() + servers_locs = pd.read_csv(SERVERS_FILE)["id"].values.tolist() return servers_locs + def get_random_input(locations): geo_input = {} for i in range(COMMITTEE_SIZE): @@ -64,34 +70,39 @@ def get_random_input(locations): geo_input[random_loc] = 1 return geo_input + def get_custom_input(majority, minority, majority_count): geo_input = {} geo_input[majority] = majority_count minority_size = COMMITTEE_SIZE - majority_count - if(minority_size > 0): + if minority_size > 0: geo_input[minority] = minority_size return geo_input + def get_custom_input_twomajorities(majority, minority, i): geo_input = {} geo_input[minority] = i - majority_size = math.floor((COMMITTEE_SIZE - i)/2) + majority_size = math.floor((COMMITTEE_SIZE - i) / 2) majority1_size = COMMITTEE_SIZE - i - majority_size geo_input[majority[0]] = majority_size geo_input[majority[1]] = majority1_size return geo_input + def get_continent_data(continent_codes): servers = pd.read_csv(SERVERS_FILE) continent_servers = servers[servers["continent"].isin(continent_codes)] - servers_locs = continent_servers['id'].values.tolist() + servers_locs = continent_servers["id"].values.tolist() return servers_locs + def get_us_europe_validators(signal): servers = pd.read_csv(MARKED_SERVERS_FILE) - selected_servers = servers[servers["is_US_Europe"]==signal] - return selected_servers['id'].values.tolist() - + selected_servers = servers[servers["is_US_Europe"] == signal] + return selected_servers["id"].values.tolist() + + def get_us_europe_rest_distribution(minority_size): geo_input = {} us_europe_ids = get_us_europe_validators(1) @@ -106,38 +117,40 @@ def get_us_europe_rest_distribution(minority_size): else: geo_input[random_loc] = 2 x = x - 2 - + # fill in the remaining seats with majority, each location has six majority_size = COMMITTEE_SIZE - minority_size while majority_size > 0: number = majority_size - if majority_size > 8 : + if majority_size > 8: number = 8 majority_size = majority_size - number - else: + else: majority_size = 0 random_loc = random.choice(us_europe_ids) - + if random_loc in geo_input.keys(): geo_input[random_loc] = number + geo_input[random_loc] else: geo_input[random_loc] = number - + return geo_input + ## this function checks if we have all the pairs of data for exsiting inputs def check_if_valid_input(geo_input, pingDelays): keys = list(geo_input.keys()) - + for source in keys: for destination in keys: if source != destination: - query = 'source == ' + str(source) + ' and destination == '+ str(destination) - delay_data = pingDelays.query(query) - if(delay_data.empty): - return False + query = "source == " + str(source) + " and destination == " + str(destination) + delay_data = pingDelays.query(query) + if delay_data.empty: + return False return True + def get_random_64node(minority_size): geo_input = {} us_europe_ids = get_us_europe_validators(1) @@ -160,17 +173,18 @@ def get_random_64node(minority_size): geo_input[random_loc] = 1 + geo_input[random_loc] else: geo_input[random_loc] = 1 - majority_size = majority_size -1 + majority_size = majority_size - 1 return geo_input + def get_fixed_64node(minority_size): geo_input = {} - + # 20 each from San Jose and Helinski geo_input[23] = 20 geo_input[45] = 20 - - # select the rest of validators. We picked these locations as we observed these two buckets in most networks. + + # select the rest of validators. We picked these locations as we observed these two buckets in most networks. us_europe_ids = get_us_europe_validators(1) non_us_europe_ids = get_us_europe_validators(0) @@ -191,13 +205,10 @@ def get_fixed_64node(minority_size): geo_input[random_loc] = 1 + geo_input[random_loc] else: geo_input[random_loc] = 1 - majority_size = majority_size -1 + majority_size = majority_size - 1 return geo_input -if __name__ == "__main__": - - - locations = get_server_locations(SERVERS_FILE) + # locations = get_server_locations(SERVERS_FILE) ################################# ##### RANDOM INPUT RUNS ######### ################################# @@ -207,7 +218,7 @@ def get_fixed_64node(minority_size): # # Get random inputs from the runs # geo_input = get_random_input(locations) # change_location_input("fabfile.py", geo_input) - + # now = datetime.datetime.now() # print("==============================================================") # print(str(now) + " Running "+ str(i) +" test with " + str(geo_input)) @@ -227,7 +238,7 @@ def get_fixed_64node(minority_size): # while(i>(COMMITTEE_SIZE/2)): # geo_input = get_custom_input(majority, minority, i) # change_location_input("fabfile.py", geo_input) - + # now = datetime.datetime.now() # print("==============================================================") # print(str(now) + " Running "+ str(i) +" test with " + str(geo_input)) @@ -237,8 +248,8 @@ def get_fixed_64node(minority_size): # print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") # sleep(1) # i = i -1 - - ####################################################### + + ####################################################### # # MINORITY IN ONE CONTINENT ######################### ####################################################### ## Majority in Helinski, Finland and Santa Clara, US @@ -249,7 +260,7 @@ def get_fixed_64node(minority_size): # while i < 3=: # geo_input = get_custom_input_twomajorities(majority, minority, i) # change_location_input("fabfile.py", geo_input) - + # now = datetime.datetime.now() # print("==============================================================") # print(str(now) + " Running "+ str(i) +" test with " + str(geo_input)) @@ -259,25 +270,25 @@ def get_fixed_64node(minority_size): # print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") # sleep(1) # i = i +1 - + # ################################################################## # ### 64 nodes. Majority in US/Europe. Keep varying the minority#### # ### We pick all nodes from these locations randomly ############## # ################################################################## # geodec = GeoDec() - + # runs = 2 - + # while runs > 0: # runs = runs - 1 - + # minority_count = 24 - + # while minority_count > 0: - + # geo_input = get_us_europe_rest_distribution(minority_count) - # pingDelays = geodec.getPingDelay(geo_input, "/home/ubuntu/data/pings-2020-07-19-2020-07-20-grouped.csv", "/home/ubuntu/data/pings-2020-07-19-2020-07-20.csv") - + # pingDelays = geodec.getPingDelay(geo_input, "/home/ubuntu/data/pings-2020-07-19-2020-07-20-grouped.csv", "/home/ubuntu/data/pings-2020-07-19-2020-07-20.csv") + # change_location_input("fabfile.py", geo_input) # if(check_if_valid_input(geo_input, pingDelays)): # now = datetime.datetime.now() @@ -288,7 +299,7 @@ def get_fixed_64node(minority_size): # print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") # sleep(1) - + # minority_count = minority_count - 4 # ################################################################## # ### 64 nodes. Majority in US/Europe. Keep varying the minority#### @@ -296,17 +307,17 @@ def get_fixed_64node(minority_size): # ################################################################## # #### 64node-random-mean-geo-dec-metrics.csv # geodec = GeoDec() - + # runs = 2 - + # while runs > 0: # runs = runs - 1 - + # minority_count = 24 - + # while minority_count > 0: # geo_input = get_random_64node(minority_count) - # pingDelays = geodec.getPingDelay(geo_input, "/home/ubuntu/data/pings-2020-07-19-2020-07-20-grouped.csv", "/home/ubuntu/data/pings-2020-07-19-2020-07-20.csv") + # pingDelays = geodec.getPingDelay(geo_input, "/home/ubuntu/data/pings-2020-07-19-2020-07-20-grouped.csv", "/home/ubuntu/data/pings-2020-07-19-2020-07-20.csv") # print("I am here") # change_location_input("fabfile.py", geo_input) @@ -320,7 +331,7 @@ def get_fixed_64node(minority_size): # print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") # sleep(1) - + # minority_count = minority_count - 4 # else: # print("ehat to do ey") @@ -329,40 +340,52 @@ def get_fixed_64node(minority_size): ### We pick all nodes from other locations randomly ######################### ################################################################## #### 64node-fixed-mean-geo-dec-metrics.csv - geodec = GeoDec() - - runs = 5 - - while runs > 0: - runs = runs - 1 - - minority_count = 24 - - while minority_count > 0: - geo_input = get_fixed_64node(minority_count) - pingDelays = geodec.getPingDelay(geo_input, "/home/ubuntu/data/pings-2020-07-19-2020-07-20-grouped.csv", "/home/ubuntu/data/pings-2020-07-19-2020-07-20.csv") - - change_location_input("fabfile.py", geo_input) - if(check_if_valid_input(geo_input, pingDelays)): - now = datetime.datetime.now() - print("==============================================================") - print(str(now) + " Running "+ str(runs) +" test with " + str(geo_input) + " Non Europe and US count is "+ str(minority_count)) - - subprocess.run(["fab", "remote"]) - - print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") - sleep(1) - - minority_count = minority_count - 4 - else: - print("eh what to do ey") - ####################################################### + # geodec = GeoDec() + + # runs = 5 + + # while runs > 0: + # runs = runs - 1 + + # minority_count = 24 + + # while minority_count > 0: + # geo_input = get_fixed_64node(minority_count) + # pingDelays = geodec.getPingDelay( + # geo_input, + # "/home/ubuntu/data/pings-2020-07-19-2020-07-20-grouped.csv", + # "/home/ubuntu/data/pings-2020-07-19-2020-07-20.csv", + # ) + + # change_location_input("fabfile.py", geo_input) + # if check_if_valid_input(geo_input, pingDelays): + # now = datetime.datetime.now() + # print("==============================================================") + # print( + # str(now) + # + " Running " + # + str(runs) + # + " test with " + # + str(geo_input) + # + " Non Europe and US count is " + # + str(minority_count) + # ) + + # subprocess.run(["fab", "remote"]) + + # print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") + # sleep(1) + + # minority_count = minority_count - 4 + # else: + # print("eh what to do ey") + ####################################################### #### BASIC RUNS ####################################### - ####################################################### - - # message_sizes = [ 16, 32] + ####################################################### + + # message_sizes = [16, 32] # batch_sizes = [200, 500, 1000, 10000, 20000, 50000, 80000, 100000] - # tgt_tp = [20000 , 30000, 50000, 100000, 200000, 450000] + # tgt_tp = [20000, 30000, 50000, 100000, 200000, 450000] # repeat = 5 # print("Starting benchmarking tool") @@ -382,4 +405,43 @@ def get_fixed_64node(minority_size): # print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") # sleep(1) - print("Benchmarking finished") \ No newline at end of file + +def update_batch_size(json_file_path, mechanism_name, new_batch_size): + try: + with open(json_file_path, "r") as file: + data = json.load(file) + + if mechanism_name in data["remote"]: + data["remote"][mechanism_name]["node_params"]["mempool"]["batch_size"] = new_batch_size + with open(json_file_path, "w") as file: + json.dump(data, file, indent=4) + print(f"Batch size for {mechanism_name} updated to {new_batch_size}.") + else: + print(f"Mechanism {mechanism_name} not found in the JSON file.") + + except Exception as e: + print(f"An error occurred: {e}") + + +if __name__ == "__main__": + + batch_sizes = [256, 512, 1024, 10000, 20000, 50000, 80000, 100000] + mechanism = ["hotstuff", "bullshark"] + + print("Starting benchmarking tool") + + for name in mechanism: + for b in batch_sizes: + # run = f"run_m{m}_b{b*m}_t{t}_repeat{i}", + now = datetime.datetime.now() + + print("==============================================================") + print(str(now) + " Running test: ") + + update_batch_size("../fab-params.json", name, b) + subprocess.run(["fab", "remote", name]) + + print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") + sleep(5) + + print("Benchmarking finished") diff --git a/fab-params.json b/fab-params.json index 95c98f8..6a1b717 100644 --- a/fab-params.json +++ b/fab-params.json @@ -6,7 +6,7 @@ "nodes": [4], "rate": [100000], "tx_size": 256, - "duration": 60, + "duration": 300, "runs": 3 }, "node_params": { @@ -27,10 +27,10 @@ "bench_params": { "faults": 0, "nodes": [4], - "rate": [1000], + "rate": [2000], "tx_size": 256, - "duration": 60, - "runs": 1 + "duration": 300, + "runs": 3 }, "node_params": { "consensus": { @@ -52,10 +52,10 @@ "nodes": [4], "workers": 1, "collocate": true, - "rate": [100000], + "rate": [200000], "tx_size": 256, - "duration": 100, - "runs": 1 + "duration": 300, + "runs": 3 }, "node_params": { "header_size": 50, From c071bf640dd0e82f425bb58863bbe9de2e2985cd Mon Sep 17 00:00:00 2001 From: Naman Garg <0708ng@gmail.com> Date: Sun, 14 Jul 2024 17:32:29 +0000 Subject: [PATCH 09/11] automated testing fix --- benchmark/geo_runs.py | 9 ++++++--- benchmark/logs.py | 6 +----- benchmark/mechanisms/bullshark.py | 2 +- fab-params.json | 30 +++++++++++++++++++++--------- results/metrics.csv | 31 ++++++++++++++++++++++++++++++- 5 files changed, 59 insertions(+), 19 deletions(-) diff --git a/benchmark/geo_runs.py b/benchmark/geo_runs.py index 5fa5012..67d69ec 100644 --- a/benchmark/geo_runs.py +++ b/benchmark/geo_runs.py @@ -412,7 +412,10 @@ def update_batch_size(json_file_path, mechanism_name, new_batch_size): data = json.load(file) if mechanism_name in data["remote"]: - data["remote"][mechanism_name]["node_params"]["mempool"]["batch_size"] = new_batch_size + if mechanism_name == "hotstuff": + data["remote"][mechanism_name]["node_params"]["mempool"]["batch_size"] = new_batch_size + elif mechanism_name == "bullshark": + data["remote"][mechanism_name]["node_params"]["batch_size"] = new_batch_size with open(json_file_path, "w") as file: json.dump(data, file, indent=4) print(f"Batch size for {mechanism_name} updated to {new_batch_size}.") @@ -426,7 +429,7 @@ def update_batch_size(json_file_path, mechanism_name, new_batch_size): if __name__ == "__main__": batch_sizes = [256, 512, 1024, 10000, 20000, 50000, 80000, 100000] - mechanism = ["hotstuff", "bullshark"] + mechanism = ["bullshark"] print("Starting benchmarking tool") @@ -438,7 +441,7 @@ def update_batch_size(json_file_path, mechanism_name, new_batch_size): print("==============================================================") print(str(now) + " Running test: ") - update_batch_size("../fab-params.json", name, b) + update_batch_size("/home/ubuntu/geodec/fab-params.json", name, b) subprocess.run(["fab", "remote", name]) print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") diff --git a/benchmark/logs.py b/benchmark/logs.py index 7a45db8..6cd3d24 100644 --- a/benchmark/logs.py +++ b/benchmark/logs.py @@ -148,11 +148,7 @@ def parse_results(self): results["execution_time"] = int(line.split(":")[1].strip().split(" ")[0]) elif line.startswith(" Mempool batch size:"): results["batch_size"] = int(line.split(":")[1].strip().split(" ")[0]) - - # Parsing the RESULTS section - for line in lines: - line = line.replace(",", "") - if line.startswith(" Consensus TPS:"): + elif line.startswith(" Consensus TPS:"): results["consensus_tps"] = int(line.split(":")[1].strip().split(" ")[0]) elif line.startswith(" Consensus BPS:"): results["consensus_bps"] = int(line.split(":")[1].strip().split(" ")[0]) diff --git a/benchmark/mechanisms/bullshark.py b/benchmark/mechanisms/bullshark.py index e1ad2f7..f78aa51 100644 --- a/benchmark/mechanisms/bullshark.py +++ b/benchmark/mechanisms/bullshark.py @@ -496,7 +496,7 @@ def result(self): f' GC depth: {gc_depth:,} round(s)\n' f' Sync retry delay: {sync_retry_delay:,} ms\n' f' Sync retry nodes: {sync_retry_nodes:,} node(s)\n' - f' batch size: {batch_size:,} B\n' + f' Mempool batch size: {batch_size:,} B\n' f' Max batch delay: {max_batch_delay:,} ms\n' '\n' ' + RESULTS:\n' diff --git a/fab-params.json b/fab-params.json index 6a1b717..d3c1902 100644 --- a/fab-params.json +++ b/fab-params.json @@ -3,8 +3,12 @@ "hotstuff": { "bench_params": { "faults": 0, - "nodes": [4], - "rate": [100000], + "nodes": [ + 4 + ], + "rate": [ + 100000 + ], "tx_size": 256, "duration": 300, "runs": 3 @@ -18,7 +22,7 @@ "gc_depth": 50, "sync_retry_delay": 5000, "sync_retry_nodes": 3, - "batch_size": 204800, + "batch_size": 100000, "max_batch_delay": 100 } } @@ -26,8 +30,12 @@ "cometbft": { "bench_params": { "faults": 0, - "nodes": [4], - "rate": [2000], + "nodes": [ + 4 + ], + "rate": [ + 2000 + ], "tx_size": 256, "duration": 300, "runs": 3 @@ -49,10 +57,14 @@ "bullshark": { "bench_params": { "faults": 0, - "nodes": [4], + "nodes": [ + 4 + ], "workers": 1, "collocate": true, - "rate": [200000], + "rate": [ + 200000 + ], "tx_size": 256, "duration": 300, "runs": 3 @@ -63,9 +75,9 @@ "gc_depth": 50, "sync_retry_delay": 5000, "sync_retry_nodes": 3, - "batch_size": 204800, + "batch_size": 256, "max_batch_delay": 100 } } } -} +} \ No newline at end of file diff --git a/results/metrics.csv b/results/metrics.csv index a60e796..a987d53 100644 --- a/results/metrics.csv +++ b/results/metrics.csv @@ -5,4 +5,33 @@ run_id,name,faults,input_rate,committee_size,transaction_size,execution_time,bat 4,hotstuff,0.0,100000.0,4.0,256.0,59.0,204800.0,100007.0,25601867.0,16.0,99934.0,25583121.0,173.0, 5,hotstuff,0.0,100000.0,4.0,256.0,60.0,204800.0,99965.0,25591072.0,38.0,99944.0,25585548.0,264.0, 6,hotstuff,0.0,100000.0,4.0,256.0,60.0,204800.0,99980.0,25594813.0,18.0,99955.0,25588454.0,242.0, -7,hotstuff,0.0,100000.0,4.0,256.0,59.0,204800.0,99984.0,25595917.333333332,24.0,99944.33333333333,25585707.666666668,226.33333333333334,3.0 +7,hotstuff,0.0,100000.0,4.0,256.0,59.0,204800.0,99984.0,25595917.33333333,24.0,99944.33333333331,25585707.666666668,226.3333333333333,3.0 +8,hotstuff,0.0,100000.0,4.0,256.0,30.0,256.0,2456.0,628645.0,20949.0,2455.0,628415.0,19554.0, +9,hotstuff,0.0,100000.0,4.0,256.0,13.0,512.0,3276.0,838679.0,8483.0,3274.0,838223.0,7987.0, +10,hotstuff,0.0,100000.0,4.0,256.0,10.0,512.0,3107.0,795437.0,8911.0,3104.0,794749.0,8851.0, +11,hotstuff,0.0,100000.0,4.0,256.0,9.0,512.0,2282.0,584219.0,6187.0,2280.0,583627.0,5807.0, +12,hotstuff,0.0,100000.0,4.0,256.0,13.0,512.0,2888.333333333333,739445.0,7860.333333333333,2886.0,738866.3333333334,7548.333333333333,3.0 +13,hotstuff,0.0,100000.0,4.0,256.0,16.0,1024.0,38050.0,9740741.0,3889.0,38016.0,9732070.0,4412.0, +14,hotstuff,0.0,100000.0,4.0,256.0,68.0,1024.0,12098.0,3097060.0,12974.0,12094.0,3096150.0,13386.0, +15,hotstuff,0.0,100000.0,4.0,256.0,12.0,1024.0,36653.0,9383215.0,1445.0,36632.0,9377838.0,1694.0, +16,hotstuff,0.0,100000.0,4.0,256.0,16.0,1024.0,28933.666666666668,7407005.333333333,6102.666666666667,28914.0,7402019.333333333,6497.333333333333,3.0 +17,hotstuff,0.0,100000.0,4.0,256.0,300.0,10000.0,99599.0,25497382.0,499.0,99597.0,25496788.0,1012.0, +18,hotstuff,0.0,100000.0,4.0,256.0,286.0,10000.0,96864.0,24797210.0,799.0,96857.0,24795387.0,1762.0, +19,hotstuff,0.0,100000.0,4.0,256.0,227.0,10000.0,98385.0,25186464.0,1541.0,98383.0,25186132.0,2840.0, +20,hotstuff,0.0,100000.0,4.0,256.0,300.0,10000.0,98282.66666666669,25160352.0,946.3333333333334,98279.0,25159435.666666668,1871.3333333333333,3.0 +21,hotstuff,0.0,100000.0,4.0,256.0,301.0,20000.0,96861.0,24796394.0,199.0,96820.0,24785917.0,498.0, +22,hotstuff,0.0,100000.0,4.0,256.0,290.0,20000.0,99692.0,25521072.0,261.0,99688.0,25520103.0,620.0, +23,hotstuff,0.0,100000.0,4.0,256.0,300.0,20000.0,100012.0,25603132.0,220.0,100007.0,25601681.0,537.0, +24,hotstuff,0.0,100000.0,4.0,256.0,301.0,20000.0,98855.0,25306866.0,226.66666666666663,98838.33333333331,25302567.0,551.6666666666666,3.0 +25,hotstuff,0.0,100000.0,4.0,256.0,301.0,50000.0,99881.0,25569536.0,23.0,99804.0,25549805.0,59.0, +26,hotstuff,0.0,100000.0,4.0,256.0,299.0,50000.0,99421.0,25451769.0,84.0,99417.0,25450832.0,112.0, +27,hotstuff,0.0,100000.0,4.0,256.0,300.0,50000.0,100006.0,25601508.0,82.0,100001.0,25600144.0,347.0, +28,hotstuff,0.0,100000.0,4.0,256.0,301.0,50000.0,99769.33333333331,25540937.666666668,63.0,99740.66666666669,25533593.666666668,172.66666666666666,3.0 +29,hotstuff,0.0,100000.0,4.0,256.0,300.0,80000.0,100014.0,25603696.0,25.0,100004.0,25601139.0,151.0, +30,hotstuff,0.0,100000.0,4.0,256.0,300.0,80000.0,99599.0,25497335.0,155.0,99594.0,25496145.0,998.0, +31,hotstuff,0.0,100000.0,4.0,256.0,300.0,80000.0,99973.0,25593185.0,10.0,99972.0,25592844.0,49.0, +32,hotstuff,0.0,100000.0,4.0,256.0,300.0,80000.0,99862.0,25564738.666666668,63.333333333333336,99856.66666666669,25563376.0,399.3333333333333,3.0 +33,hotstuff,0.0,100000.0,4.0,256.0,301.0,100000.0,100008.0,25602007.0,82.0,100005.0,25601240.0,447.0, +34,hotstuff,0.0,100000.0,4.0,256.0,300.0,100000.0,100006.0,25601576.0,34.0,100004.0,25601065.0,286.0, +35,hotstuff,0.0,100000.0,4.0,256.0,300.0,100000.0,100008.0,25602058.0,27.0,100004.0,25601119.0,130.0, +36,hotstuff,0.0,100000.0,4.0,256.0,301.0,100000.0,100007.33333333331,25601880.33333333,47.66666666666666,100004.33333333331,25601141.33333333,287.6666666666667,3.0 From eb51c3b5b7b12f1ca7d14863a0b65936ae7800c6 Mon Sep 17 00:00:00 2001 From: Naman Garg <0708ng@gmail.com> Date: Sun, 14 Jul 2024 21:04:52 +0000 Subject: [PATCH 10/11] geo run 1 complete --- benchmark/geo_runs.py | 6 +++--- fab-params.json | 6 +++--- results/metrics.csv | 29 +++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/benchmark/geo_runs.py b/benchmark/geo_runs.py index 67d69ec..71e2aab 100644 --- a/benchmark/geo_runs.py +++ b/benchmark/geo_runs.py @@ -428,8 +428,8 @@ def update_batch_size(json_file_path, mechanism_name, new_batch_size): if __name__ == "__main__": - batch_sizes = [256, 512, 1024, 10000, 20000, 50000, 80000, 100000] - mechanism = ["bullshark"] + batch_sizes = [512, 1024, 10000, 20000, 50000, 80000, 100000, 200000, 300000] + mechanism = ["hotstuff", "bullshark"] print("Starting benchmarking tool") @@ -445,6 +445,6 @@ def update_batch_size(json_file_path, mechanism_name, new_batch_size): subprocess.run(["fab", "remote", name]) print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") - sleep(5) + sleep(10) print("Benchmarking finished") diff --git a/fab-params.json b/fab-params.json index d3c1902..603dab7 100644 --- a/fab-params.json +++ b/fab-params.json @@ -22,7 +22,7 @@ "gc_depth": 50, "sync_retry_delay": 5000, "sync_retry_nodes": 3, - "batch_size": 100000, + "batch_size": 512, "max_batch_delay": 100 } } @@ -63,7 +63,7 @@ "workers": 1, "collocate": true, "rate": [ - 200000 + 150000 ], "tx_size": 256, "duration": 300, @@ -75,7 +75,7 @@ "gc_depth": 50, "sync_retry_delay": 5000, "sync_retry_nodes": 3, - "batch_size": 256, + "batch_size": 100000, "max_batch_delay": 100 } } diff --git a/results/metrics.csv b/results/metrics.csv index a987d53..f75a8e7 100644 --- a/results/metrics.csv +++ b/results/metrics.csv @@ -35,3 +35,32 @@ run_id,name,faults,input_rate,committee_size,transaction_size,execution_time,bat 34,hotstuff,0.0,100000.0,4.0,256.0,300.0,100000.0,100006.0,25601576.0,34.0,100004.0,25601065.0,286.0, 35,hotstuff,0.0,100000.0,4.0,256.0,300.0,100000.0,100008.0,25602058.0,27.0,100004.0,25601119.0,130.0, 36,hotstuff,0.0,100000.0,4.0,256.0,301.0,100000.0,100007.33333333331,25601880.33333333,47.66666666666666,100004.33333333331,25601141.33333333,287.6666666666667,3.0 +37,bullshark,0.0,200000.0,4.0,256.0,8.0,256.0,7889.0,2019512.0,168.0,7884.0,2018313.0,1218.0, +38,bullshark,0.0,200000.0,4.0,256.0,23.0,256.0,7087.0,1814355.0,1046.0,7085.0,1813875.0,2447.0, +39,bullshark,0.0,200000.0,4.0,256.0,26.0,256.0,4904.0,1255343.0,3330.0,4903.0,1255058.0,3692.0, +40,bullshark,0.0,200000.0,4.0,256.0,8.0,256.0,6626.666666666667,1696403.3333333333,1514.6666666666667,6624.0,1695748.6666666667,2452.333333333333,3.0 +41,bullshark,0.0,200000.0,4.0,256.0,1.0,512.0,27509.0,7042269.0,34.0,27363.0,7005008.0,435.0, +42,bullshark,0.0,200000.0,4.0,256.0,0.0,512.0,0.0,0.0,0.0,0.0,0.0,0.0, +43,bullshark,0.0,200000.0,4.0,256.0,0.0,512.0,0.0,0.0,0.0,0.0,0.0,0.0, +44,bullshark,0.0,200000.0,4.0,256.0,1.0,512.0,9169.666666666666,2347423.0,11.333333333333334,9121.0,2335002.6666666665,145.0,3.0 +45,bullshark,0.0,200000.0,4.0,256.0,20.0,1024.0,3705.0,948541.0,12112.0,3704.0,948309.0,15099.0, +46,bullshark,0.0,200000.0,4.0,256.0,36.0,10000.0,174200.0,44595240.0,411.0,174181.0,44590229.0,2706.0, +47,bullshark,0.0,200000.0,4.0,256.0,0.0,10000.0,0.0,0.0,0.0,0.0,0.0,0.0, +48,bullshark,0.0,200000.0,4.0,256.0,0.0,10000.0,24615.0,6301540.0,159.0,24340.0,6231051.0,223.0, +49,bullshark,0.0,200000.0,4.0,256.0,36.0,10000.0,66271.66666666667,16965593.333333332,190.0,66173.66666666667,16940426.666666668,976.3333333333334,3.0 +50,bullshark,0.0,200000.0,4.0,256.0,44.0,20000.0,151430.0,38766033.0,162.0,151413.0,38761634.0,2446.0, +51,bullshark,0.0,200000.0,4.0,256.0,0.0,20000.0,0.0,0.0,0.0,0.0,0.0,0.0, +52,bullshark,0.0,200000.0,4.0,256.0,22.0,20000.0,20960.0,5365707.0,10302.0,20954.0,5364251.0,15551.0, +53,bullshark,0.0,200000.0,4.0,256.0,44.0,20000.0,57463.333333333336,14710580.0,3488.0,57455.66666666666,14708628.333333334,5999.0,3.0 +54,bullshark,0.0,200000.0,4.0,256.0,137.0,50000.0,168371.0,43102950.0,588.0,168365.0,43101374.0,3025.0, +55,bullshark,0.0,200000.0,4.0,256.0,101.0,50000.0,124531.0,31880033.0,230.0,124525.0,31878451.0,2617.0, +56,bullshark,0.0,200000.0,4.0,256.0,21.0,50000.0,174762.0,44739190.0,97.0,174712.0,44726304.0,2512.0, +57,bullshark,0.0,200000.0,4.0,256.0,137.0,50000.0,155888.0,39907391.0,305.0,155867.33333333334,39902043.0,2718.0,3.0 +58,bullshark,0.0,200000.0,4.0,256.0,152.0,80000.0,180061.0,46095547.0,862.0,180050.0,46092815.0,3298.0, +59,bullshark,0.0,200000.0,4.0,256.0,94.0,80000.0,177587.0,45462322.0,1479.0,177574.0,45458935.0,3939.0, +60,bullshark,0.0,200000.0,4.0,256.0,93.0,80000.0,180191.0,46128890.0,856.0,180177.0,46125415.0,3309.0, +61,bullshark,0.0,200000.0,4.0,256.0,152.0,80000.0,179279.66666666666,45895586.333333336,1065.6666666666667,179267.0,45892388.333333336,3515.333333333333,3.0 +62,bullshark,0.0,200000.0,4.0,256.0,153.0,100000.0,181785.0,46536864.0,1140.0,181772.0,46533512.0,3559.0, +63,bullshark,0.0,200000.0,4.0,256.0,0.0,100000.0,111715.0,28598982.0,21.0,82316.0,21072915.0,36.0, +64,bullshark,0.0,200000.0,4.0,256.0,0.0,100000.0,142182.0,36398554.0,28.0,126811.0,32463619.0,35.0, +65,bullshark,0.0,200000.0,4.0,256.0,153.0,100000.0,145227.33333333334,37178133.333333336,396.3333333333333,130299.66666666667,33356682.0,1210.0,3.0 From 0d49b1c76d35358a5f618666e042739031ffe60d Mon Sep 17 00:00:00 2001 From: Naman Garg <0708ng@gmail.com> Date: Mon, 15 Jul 2024 05:28:02 +0000 Subject: [PATCH 11/11] geo run 2 done --- fab-params.json | 4 +-- results/metrics.csv | 70 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/fab-params.json b/fab-params.json index 603dab7..ef50298 100644 --- a/fab-params.json +++ b/fab-params.json @@ -22,7 +22,7 @@ "gc_depth": 50, "sync_retry_delay": 5000, "sync_retry_nodes": 3, - "batch_size": 512, + "batch_size": 300000, "max_batch_delay": 100 } } @@ -75,7 +75,7 @@ "gc_depth": 50, "sync_retry_delay": 5000, "sync_retry_nodes": 3, - "batch_size": 100000, + "batch_size": 300000, "max_batch_delay": 100 } } diff --git a/results/metrics.csv b/results/metrics.csv index f75a8e7..21299b4 100644 --- a/results/metrics.csv +++ b/results/metrics.csv @@ -63,4 +63,72 @@ run_id,name,faults,input_rate,committee_size,transaction_size,execution_time,bat 62,bullshark,0.0,200000.0,4.0,256.0,153.0,100000.0,181785.0,46536864.0,1140.0,181772.0,46533512.0,3559.0, 63,bullshark,0.0,200000.0,4.0,256.0,0.0,100000.0,111715.0,28598982.0,21.0,82316.0,21072915.0,36.0, 64,bullshark,0.0,200000.0,4.0,256.0,0.0,100000.0,142182.0,36398554.0,28.0,126811.0,32463619.0,35.0, -65,bullshark,0.0,200000.0,4.0,256.0,153.0,100000.0,145227.33333333334,37178133.333333336,396.3333333333333,130299.66666666667,33356682.0,1210.0,3.0 +65,bullshark,0.0,200000.0,4.0,256.0,153.0,100000.0,145227.33333333334,37178133.333333336,396.3333333333333,130299.66666666669,33356682.0,1210.0,3.0 +66,hotstuff,0.0,100000.0,4.0,256.0,4.0,512.0,5642.0,1444280.0,2782.0,5626.0,1440316.0,2825.0, +67,hotstuff,0.0,100000.0,4.0,256.0,4.0,512.0,3884.0,994309.0,3503.0,3874.0,991856.0,3385.0, +68,hotstuff,0.0,100000.0,4.0,256.0,9.0,512.0,5630.0,1441348.0,6474.0,5607.0,1435351.0,6631.0, +69,hotstuff,0.0,100000.0,4.0,256.0,4.0,512.0,5052.0,1293312.3333333333,4253.0,5035.666666666667,1289174.3333333333,4280.333333333333,3.0 +70,hotstuff,0.0,100000.0,4.0,256.0,7.0,1024.0,20311.0,5199718.0,5285.0,20278.0,5191284.0,5979.0, +71,hotstuff,0.0,100000.0,4.0,256.0,15.0,1024.0,33813.0,8656032.0,4342.0,33791.0,8650385.0,4921.0, +72,hotstuff,0.0,100000.0,4.0,256.0,8.0,1024.0,30097.0,7704899.0,1927.0,30090.0,7703017.0,2174.0, +73,hotstuff,0.0,100000.0,4.0,256.0,7.0,1024.0,28073.666666666668,7186883.0,3851.3333333333335,28053.0,7181562.0,4358.0,3.0 +74,hotstuff,0.0,100000.0,4.0,256.0,301.0,10000.0,93007.0,23809728.0,900.0,93006.0,23809411.0,2021.0, +75,hotstuff,0.0,100000.0,4.0,256.0,299.0,10000.0,88881.0,22753513.0,1076.0,88878.0,22752753.0,2752.0, +76,hotstuff,0.0,100000.0,4.0,256.0,231.0,10000.0,55124.0,14111857.0,12217.0,55121.0,14110879.0,15907.0, +77,hotstuff,0.0,100000.0,4.0,256.0,301.0,10000.0,79004.0,20225032.666666668,4731.0,79001.66666666667,20224347.666666668,6893.333333333333,3.0 +78,hotstuff,0.0,100000.0,4.0,256.0,300.0,20000.0,100006.0,25601557.0,43.0,100005.0,25601216.0,248.0, +79,hotstuff,0.0,100000.0,4.0,256.0,300.0,20000.0,100007.0,25601884.0,160.0,100005.0,25601288.0,536.0, +80,hotstuff,0.0,100000.0,4.0,256.0,300.0,20000.0,100004.0,25600933.0,14.0,100002.0,25600592.0,54.0, +81,hotstuff,0.0,100000.0,4.0,256.0,300.0,20000.0,100005.66666666669,25601458.0,72.33333333333333,100004.0,25601032.0,279.3333333333333,3.0 +82,hotstuff,0.0,100000.0,4.0,256.0,300.0,50000.0,100007.0,25601787.0,22.0,100004.0,25601105.0,180.0, +83,hotstuff,0.0,100000.0,4.0,256.0,300.0,50000.0,100009.0,25602220.0,33.0,100005.0,25601281.0,144.0, +84,hotstuff,0.0,100000.0,4.0,256.0,300.0,50000.0,100004.0,25601134.0,61.0,100001.0,25600281.0,393.0, +85,hotstuff,0.0,100000.0,4.0,256.0,300.0,50000.0,100006.66666666669,25601713.666666668,38.66666666666666,100003.33333333331,25600889.0,239.0,3.0 +86,hotstuff,0.0,100000.0,4.0,256.0,301.0,80000.0,100001.0,25600373.0,7.0,99998.0,25599436.0,16.0, +87,hotstuff,0.0,100000.0,4.0,256.0,300.0,80000.0,100010.0,25602481.0,28.0,100008.0,25601970.0,91.0, +88,hotstuff,0.0,100000.0,4.0,256.0,300.0,80000.0,100003.0,25600642.0,195.0,100002.0,25600471.0,886.0, +89,hotstuff,0.0,100000.0,4.0,256.0,301.0,80000.0,100004.66666666669,25601165.33333333,76.66666666666667,100002.66666666669,25600625.666666668,331.0,3.0 +90,hotstuff,0.0,100000.0,4.0,256.0,300.0,100000.0,100026.0,25606649.0,51.0,100006.0,25601617.0,185.0, +91,hotstuff,0.0,100000.0,4.0,256.0,300.0,100000.0,100012.0,25603187.0,101.0,100009.0,25602249.0,377.0, +92,hotstuff,0.0,100000.0,4.0,256.0,300.0,100000.0,99970.0,25592212.0,115.0,99968.0,25591785.0,618.0, +93,hotstuff,0.0,100000.0,4.0,256.0,300.0,100000.0,100002.66666666669,25600682.666666668,89.0,99994.33333333331,25598550.33333333,393.3333333333333,3.0 +94,hotstuff,0.0,100000.0,4.0,256.0,300.0,200000.0,100000.0,25599895.0,26.0,99995.0,25598617.0,84.0, +95,hotstuff,0.0,100000.0,4.0,256.0,300.0,200000.0,100009.0,25602202.0,37.0,99988.0,25596832.0,73.0, +96,hotstuff,0.0,100000.0,4.0,256.0,300.0,200000.0,100019.0,25604808.0,44.0,99999.0,25599865.0,205.0, +97,hotstuff,0.0,100000.0,4.0,256.0,300.0,200000.0,100009.33333333331,25602301.666666668,35.666666666666664,99994.0,25598438.0,120.66666666666669,3.0 +98,hotstuff,0.0,100000.0,4.0,256.0,300.0,300000.0,99990.0,25597351.0,61.0,99985.0,25596073.0,183.0, +99,hotstuff,0.0,100000.0,4.0,256.0,301.0,300000.0,100005.0,25601245.0,57.0,99997.0,25599286.0,416.0, +100,hotstuff,0.0,100000.0,4.0,256.0,300.0,300000.0,99999.0,25599845.0,66.0,99996.0,25599078.0,662.0, +101,hotstuff,0.0,100000.0,4.0,256.0,300.0,300000.0,99998.0,25599480.33333333,61.333333333333336,99992.66666666669,25598145.666666668,420.3333333333333,3.0 +102,bullshark,0.0,150000.0,4.0,256.0,19.0,512.0,21394.0,5476925.0,772.0,21389.0,5475486.0,4474.0, +103,bullshark,0.0,150000.0,4.0,256.0,42.0,512.0,35004.0,8960908.0,1087.0,35000.0,8960055.0,4518.0, +104,bullshark,0.0,150000.0,4.0,256.0,48.0,512.0,28414.0,7274017.0,1626.0,28409.0,7272801.0,6103.0, +105,bullshark,0.0,150000.0,4.0,256.0,19.0,512.0,28270.666666666668,7237283.333333333,1161.6666666666667,28266.0,7236114.0,5031.666666666667,3.0 +106,bullshark,0.0,150000.0,4.0,256.0,169.0,10000.0,123926.0,31725145.0,315.0,123923.0,31724396.0,2612.0, +107,bullshark,0.0,150000.0,4.0,256.0,0.0,10000.0,40000.0,10240016.0,13.0,37949.0,9714863.0,16.0, +108,bullshark,0.0,150000.0,4.0,256.0,33.0,10000.0,25008.0,6402120.0,3156.0,25002.0,6400561.0,6368.0, +109,bullshark,0.0,150000.0,4.0,256.0,169.0,10000.0,62978.0,16122427.0,1161.3333333333333,62291.333333333336,15946606.666666666,2998.6666666666665,3.0 +110,bullshark,0.0,150000.0,4.0,256.0,175.0,20000.0,127637.0,32675112.0,157.0,127631.0,32673620.0,2543.0, +111,bullshark,0.0,150000.0,4.0,256.0,10.0,20000.0,58913.0,15081795.0,493.0,58867.0,15069845.0,2912.0, +112,bullshark,0.0,150000.0,4.0,256.0,0.0,20000.0,0.0,0.0,0.0,0.0,0.0,0.0, +113,bullshark,0.0,150000.0,4.0,256.0,175.0,20000.0,62183.333333333336,15918969.0,216.66666666666663,62166.0,15914488.333333334,1818.3333333333333,3.0 +114,bullshark,0.0,150000.0,4.0,256.0,93.0,50000.0,133503.0,34176738.0,781.0,133488.0,34173047.0,3225.0, +115,bullshark,0.0,150000.0,4.0,256.0,21.0,50000.0,23034.0,5896656.0,14757.0,23028.0,5895266.0,18781.0, +116,bullshark,0.0,150000.0,4.0,256.0,88.0,50000.0,130478.0,33402460.0,507.0,130469.0,33400174.0,2948.0, +117,bullshark,0.0,150000.0,4.0,256.0,93.0,50000.0,95671.66666666669,24491951.33333333,5348.333333333333,95661.66666666669,24489495.666666668,8318.0,3.0 +118,bullshark,0.0,150000.0,4.0,256.0,144.0,80000.0,129112.0,33052779.0,1418.0,129103.0,33050491.0,3843.0, +119,bullshark,0.0,150000.0,4.0,256.0,26.0,80000.0,66272.0,16965631.0,6703.0,66256.0,16961648.0,16200.0, +120,bullshark,0.0,150000.0,4.0,256.0,32.0,80000.0,137912.0,35305542.0,325.0,137878.0,35296739.0,2677.0, +121,bullshark,0.0,150000.0,4.0,256.0,144.0,80000.0,111098.66666666669,28441317.33333333,2815.333333333333,111079.0,28436292.666666668,7573.333333333333,3.0 +122,bullshark,0.0,150000.0,4.0,256.0,260.0,100000.0,139820.0,35793873.0,1452.0,139815.0,35792634.0,3872.0, +123,bullshark,0.0,150000.0,4.0,256.0,0.0,100000.0,128472.0,32888717.0,39.0,109671.0,28075701.0,46.0, +124,bullshark,0.0,150000.0,4.0,256.0,213.0,100000.0,133043.0,34059118.0,1558.0,133037.0,34057362.0,4008.0, +125,bullshark,0.0,150000.0,4.0,256.0,260.0,100000.0,133778.33333333334,34247236.0,1016.3333333333334,127507.66666666669,32641899.0,2642.0,3.0 +126,bullshark,0.0,150000.0,4.0,256.0,292.0,200000.0,143567.0,36753074.0,1495.0,143559.0,36751062.0,3872.0, +127,bullshark,0.0,150000.0,4.0,256.0,0.0,200000.0,0.0,0.0,0.0,0.0,0.0,0.0, +128,bullshark,0.0,150000.0,4.0,256.0,243.0,200000.0,133951.0,34291453.0,3951.0,133944.0,34289617.0,6369.0, +129,bullshark,0.0,150000.0,4.0,256.0,292.0,200000.0,92506.0,23681509.0,1815.3333333333333,92501.0,23680226.33333333,3413.6666666666665,3.0 +130,bullshark,0.0,150000.0,4.0,256.0,300.0,300000.0,148225.0,37945668.0,2700.0,148195.0,37937819.0,5064.0, +131,bullshark,0.0,150000.0,4.0,256.0,298.0,300000.0,146328.0,37459903.0,2554.0,146319.0,37457637.0,5878.0, +132,bullshark,0.0,150000.0,4.0,256.0,298.0,300000.0,139315.0,35664530.0,2403.0,139287.0,35657470.0,5661.0, +133,bullshark,0.0,150000.0,4.0,256.0,300.0,300000.0,144622.66666666666,37023367.0,2552.3333333333335,144600.33333333334,37017642.0,5534.333333333333,3.0