From 753a1aec03d39978c36970c7d07a756ddde3bc6d Mon Sep 17 00:00:00 2001 From: "jones.wz" Date: Tue, 12 Apr 2022 20:50:53 +0800 Subject: [PATCH 1/5] added a demo for black-box optimization --- demo/bbo.py | 99 ++++++++++++++++++++++++++++++++++++++++++ federatedscope/main.py | 24 +++++----- 2 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 demo/bbo.py diff --git a/demo/bbo.py b/demo/bbo.py new file mode 100644 index 000000000..6eeea2c60 --- /dev/null +++ b/demo/bbo.py @@ -0,0 +1,99 @@ +"""This python script is provided to demonstrate the interaction between emukit and FederatedScope. +Specifically, we apply Black-Box Optimization (BBO) to search the optimal hyperparameters of the considered federated learning algorithms. +emukit can be installed by `pip install emukit` +""" +import numpy as np +import matplotlib.pyplot as plt +from matplotlib import colors as mcolors + +from emukit.test_functions import forrester_function +from emukit.core import ContinuousParameter, CategoricalParameter, ParameterSpace +from emukit.examples.gp_bayesian_optimization.single_objective_bayesian_optimization import GPBayesianOptimization + +### --- Figure config +LEGEND_SIZE = 15 + + +def eval_fl_algo(x): + print("======> try {}".format(x)) + + from federatedscope.core.cmd_args import parse_args + from federatedscope.core.auxiliaries.data_builder import get_data + from federatedscope.core.auxiliaries.utils import setup_seed, setup_logger + from federatedscope.core.auxiliaries.worker_builder import get_client_cls, get_server_cls + from federatedscope.core.configs.config import global_cfg + from federatedscope.core.fed_runner import FedRunner + + init_cfg = global_cfg.clone() + init_cfg.merge_from_file("federatedscope/example_configs/single_process.yaml") + init_cfg.merge_from_list(["optimizer.lr", float(x[0])]) + + setup_logger(init_cfg) + setup_seed(init_cfg.seed) + + # federated dataset might change the number of clients + # thus, we allow the creation procedure of dataset to modify the global cfg object + data, modified_cfg = get_data(config=init_cfg.clone()) + init_cfg.merge_from_other_cfg(modified_cfg) + + init_cfg.freeze() + + runner = FedRunner(data=data, + server_class=get_server_cls(init_cfg), + client_class=get_client_cls(init_cfg), + config=init_cfg.clone()) + results = runner.run() + + # so that we could modify cfg in the next trial + init_cfg.defrost() + + return [results['client_summarized_weighted_avg']['test_avg_loss']] + + +def our_target_func(x): + return np.asarray([eval_fl_algo(elem) for elem in x]) + + +def main(): + #target_function, space = forrester_function() + target_function = our_target_func + space = ParameterSpace([ContinuousParameter('lr', 1e-4, .75)]) + x_plot = np.linspace(space.parameters[0].min, space.parameters[0].max, 200)[:, None] + #y_plot = target_function(x_plot) + X_init = np.array([[0.005],[0.05], [0.5]]) + Y_init = target_function(X_init) + + bo = GPBayesianOptimization(variables_list=space.parameters, + X=X_init, Y=Y_init) + bo.run_optimization(target_function, 15) + + mu_plot, var_plot = bo.model.predict(x_plot) + + plt.figure(figsize=(12, 8)) + plt.plot(bo.loop_state.X, bo.loop_state.Y, "ro", markersize=10, label="Observations") + #plt.plot(x_plot, y_plot, "k", label="Objective Function") + #plt.plot(x_plot, mu_plot, "C0", label="Model") + plt.fill_between(x_plot[:, 0], + mu_plot[:, 0] + np.sqrt(var_plot)[:, 0], + mu_plot[:, 0] - np.sqrt(var_plot)[:, 0], color="C0", alpha=0.6) + + plt.fill_between(x_plot[:, 0], + mu_plot[:, 0] + 2 * np.sqrt(var_plot)[:, 0], + mu_plot[:, 0] - 2 * np.sqrt(var_plot)[:, 0], color="C0", alpha=0.4) + + plt.fill_between(x_plot[:, 0], + mu_plot[:, 0] + 3 * np.sqrt(var_plot)[:, 0], + mu_plot[:, 0] - 3 * np.sqrt(var_plot)[:, 0], color="C0", alpha=0.2) + plt.legend(loc=2, prop={'size': LEGEND_SIZE}) + plt.xlabel(r"$x$") + plt.ylabel(r"$f(x)$") + plt.grid(True) + plt.xlim(0, 0.75) + + #plt.show() + plt.savefig("bbo.pdf", bbox_inches='tight') + plt.close() + + +if __name__=="__main__": + main() diff --git a/federatedscope/main.py b/federatedscope/main.py index 2c70f2469..f7d39443f 100644 --- a/federatedscope/main.py +++ b/federatedscope/main.py @@ -1,7 +1,7 @@ import os import sys -DEV_MODE = True # simplify the federatedscope re-setup everytime we change the source codes of federatedscope +DEV_MODE = False # simplify the federatedscope re-setup everytime we change the source codes of federatedscope if DEV_MODE: file_dir = os.path.join(os.path.dirname(__file__), '..') sys.path.append(file_dir) @@ -19,23 +19,23 @@ del os.environ['http_proxy'] if __name__ == '__main__': - + init_cfg = global_cfg.clone() args = parse_args() - global_cfg.merge_from_file(args.cfg_file) - global_cfg.merge_from_list(args.opts) + init_cfg.merge_from_file(args.cfg_file) + init_cfg.merge_from_list(args.opts) - setup_logger(global_cfg) - setup_seed(global_cfg.seed) + setup_logger(init_cfg) + setup_seed(init_cfg.seed) # federated dataset might change the number of clients # thus, we allow the creation procedure of dataset to modify the global cfg object - data, modified_cfg = get_data(config=global_cfg.clone()) - global_cfg.merge_from_other_cfg(modified_cfg) + data, modified_cfg = get_data(config=init_cfg.clone()) + init_cfg.merge_from_other_cfg(modified_cfg) - global_cfg.freeze() + init_cfg.freeze() runner = FedRunner(data=data, - server_class=get_server_cls(global_cfg), - client_class=get_client_cls(global_cfg), - config=global_cfg.clone()) + server_class=get_server_cls(init_cfg), + client_class=get_client_cls(init_cfg), + config=init_cfg.clone()) _ = runner.run() From 4a3567c6ce95dae4a953ecbb83ad92c35c4e7375 Mon Sep 17 00:00:00 2001 From: "jones.wz" Date: Wed, 13 Apr 2022 15:32:30 +0800 Subject: [PATCH 2/5] enabled installation with cuda10 --- .../requirements-torch1.8-application.txt | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 enviroment/requirements-torch1.8-application.txt diff --git a/enviroment/requirements-torch1.8-application.txt b/enviroment/requirements-torch1.8-application.txt new file mode 100644 index 000000000..6f52a6825 --- /dev/null +++ b/enviroment/requirements-torch1.8-application.txt @@ -0,0 +1,23 @@ +numpy==1.19.5 +scikit-learn==1.0 +scipy==1.6.0 +pandas==1.2.1 +scikit-learn +pytorch==1.8.0 +torchvision==0.9.0 +torchaudio==0.8.0 +cudatoolkit==10.2.89 +wandb +tensorboard +tensorboardX +grpcio +grpcio-tools +protobuf==3.19.1 +setuptools==58.0.4 +pyg==2.0.1 +rdkit=2021.09.4 +sentencepiece +textgrid +typeguard + + From 303b27ec8330678d5e4381fe832ae94646d9f279 Mon Sep 17 00:00:00 2001 From: "jones.wz" Date: Fri, 15 Apr 2022 18:01:13 +0800 Subject: [PATCH 3/5] updated accordingly --- demo/bbo.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/demo/bbo.py b/demo/bbo.py index 6eeea2c60..a31d64626 100644 --- a/demo/bbo.py +++ b/demo/bbo.py @@ -2,6 +2,8 @@ Specifically, we apply Black-Box Optimization (BBO) to search the optimal hyperparameters of the considered federated learning algorithms. emukit can be installed by `pip install emukit` """ +import logging + import numpy as np import matplotlib.pyplot as plt from matplotlib import colors as mcolors @@ -15,7 +17,7 @@ def eval_fl_algo(x): - print("======> try {}".format(x)) + logging.info("======> try {}".format(x)) from federatedscope.core.cmd_args import parse_args from federatedscope.core.auxiliaries.data_builder import get_data From f342edbf21cf30fd83161718319bfb05fe70690a Mon Sep 17 00:00:00 2001 From: "jones.wz" Date: Fri, 15 Apr 2022 18:09:47 +0800 Subject: [PATCH 4/5] updated accordingly --- demo/bbo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/demo/bbo.py b/demo/bbo.py index a31d64626..784653e84 100644 --- a/demo/bbo.py +++ b/demo/bbo.py @@ -17,8 +17,6 @@ def eval_fl_algo(x): - logging.info("======> try {}".format(x)) - from federatedscope.core.cmd_args import parse_args from federatedscope.core.auxiliaries.data_builder import get_data from federatedscope.core.auxiliaries.utils import setup_seed, setup_logger @@ -32,6 +30,7 @@ def eval_fl_algo(x): setup_logger(init_cfg) setup_seed(init_cfg.seed) + logging.info("======> try {}".format(x)) # federated dataset might change the number of clients # thus, we allow the creation procedure of dataset to modify the global cfg object From 21f00a0be3775077ee9391d5b5d64993fa61e0e6 Mon Sep 17 00:00:00 2001 From: joneswong Date: Sun, 17 Apr 2022 23:53:10 +0800 Subject: [PATCH 5/5] re-formatted --- demo/bbo.py | 37 +++++++++++++++++++++++++------------ federatedscope/main.py | 2 +- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/demo/bbo.py b/demo/bbo.py index 784653e84..ffe8308b0 100644 --- a/demo/bbo.py +++ b/demo/bbo.py @@ -25,7 +25,8 @@ def eval_fl_algo(x): from federatedscope.core.fed_runner import FedRunner init_cfg = global_cfg.clone() - init_cfg.merge_from_file("federatedscope/example_configs/single_process.yaml") + init_cfg.merge_from_file( + "federatedscope/example_configs/single_process.yaml") init_cfg.merge_from_list(["optimizer.lr", float(x[0])]) setup_logger(init_cfg) @@ -59,42 +60,54 @@ def main(): #target_function, space = forrester_function() target_function = our_target_func space = ParameterSpace([ContinuousParameter('lr', 1e-4, .75)]) - x_plot = np.linspace(space.parameters[0].min, space.parameters[0].max, 200)[:, None] + x_plot = np.linspace(space.parameters[0].min, space.parameters[0].max, + 200)[:, None] #y_plot = target_function(x_plot) - X_init = np.array([[0.005],[0.05], [0.5]]) + X_init = np.array([[0.005], [0.05], [0.5]]) Y_init = target_function(X_init) bo = GPBayesianOptimization(variables_list=space.parameters, - X=X_init, Y=Y_init) + X=X_init, + Y=Y_init) bo.run_optimization(target_function, 15) mu_plot, var_plot = bo.model.predict(x_plot) plt.figure(figsize=(12, 8)) - plt.plot(bo.loop_state.X, bo.loop_state.Y, "ro", markersize=10, label="Observations") + plt.plot(bo.loop_state.X, + bo.loop_state.Y, + "ro", + markersize=10, + label="Observations") #plt.plot(x_plot, y_plot, "k", label="Objective Function") #plt.plot(x_plot, mu_plot, "C0", label="Model") plt.fill_between(x_plot[:, 0], mu_plot[:, 0] + np.sqrt(var_plot)[:, 0], - mu_plot[:, 0] - np.sqrt(var_plot)[:, 0], color="C0", alpha=0.6) - + mu_plot[:, 0] - np.sqrt(var_plot)[:, 0], + color="C0", + alpha=0.6) + plt.fill_between(x_plot[:, 0], mu_plot[:, 0] + 2 * np.sqrt(var_plot)[:, 0], - mu_plot[:, 0] - 2 * np.sqrt(var_plot)[:, 0], color="C0", alpha=0.4) - + mu_plot[:, 0] - 2 * np.sqrt(var_plot)[:, 0], + color="C0", + alpha=0.4) + plt.fill_between(x_plot[:, 0], mu_plot[:, 0] + 3 * np.sqrt(var_plot)[:, 0], - mu_plot[:, 0] - 3 * np.sqrt(var_plot)[:, 0], color="C0", alpha=0.2) + mu_plot[:, 0] - 3 * np.sqrt(var_plot)[:, 0], + color="C0", + alpha=0.2) plt.legend(loc=2, prop={'size': LEGEND_SIZE}) plt.xlabel(r"$x$") plt.ylabel(r"$f(x)$") plt.grid(True) plt.xlim(0, 0.75) - + #plt.show() plt.savefig("bbo.pdf", bbox_inches='tight') plt.close() -if __name__=="__main__": +if __name__ == "__main__": main() diff --git a/federatedscope/main.py b/federatedscope/main.py index f7d39443f..b59d98edd 100644 --- a/federatedscope/main.py +++ b/federatedscope/main.py @@ -1,7 +1,7 @@ import os import sys -DEV_MODE = False # simplify the federatedscope re-setup everytime we change the source codes of federatedscope +DEV_MODE = False # simplify the federatedscope re-setup everytime we change the source codes of federatedscope if DEV_MODE: file_dir = os.path.join(os.path.dirname(__file__), '..') sys.path.append(file_dir)