From 25ce3ca6bc2d1b84e56e47675ebe4a0a129e750c Mon Sep 17 00:00:00 2001 From: Jones Wong Date: Mon, 18 Apr 2022 19:18:54 +0800 Subject: [PATCH] added a demo for black-box optimization (#14) - added a demo for black-box optimization - enabled installation with cuda10 --- demo/bbo.py | 113 ++++++++++++++++++ .../requirements-torch1.8-application.txt | 23 ++++ federatedscope/main.py | 24 ++-- 3 files changed, 148 insertions(+), 12 deletions(-) create mode 100644 demo/bbo.py create mode 100644 enviroment/requirements-torch1.8-application.txt diff --git a/demo/bbo.py b/demo/bbo.py new file mode 100644 index 000000000..ffe8308b0 --- /dev/null +++ b/demo/bbo.py @@ -0,0 +1,113 @@ +"""This python script is provided to demonstrate the interaction between emukit and FederatedScope. +Specifically, we apply Black-Box Optimization (BBO) to search the optimal hyperparameters of the considered federated learning algorithms. +emukit can be installed by `pip install emukit` +""" +import logging + +import numpy as np +import matplotlib.pyplot as plt +from matplotlib import colors as mcolors + +from emukit.test_functions import forrester_function +from emukit.core import ContinuousParameter, CategoricalParameter, ParameterSpace +from emukit.examples.gp_bayesian_optimization.single_objective_bayesian_optimization import GPBayesianOptimization + +### --- Figure config +LEGEND_SIZE = 15 + + +def eval_fl_algo(x): + from federatedscope.core.cmd_args import parse_args + from federatedscope.core.auxiliaries.data_builder import get_data + from federatedscope.core.auxiliaries.utils import setup_seed, setup_logger + from federatedscope.core.auxiliaries.worker_builder import get_client_cls, get_server_cls + from federatedscope.core.configs.config import global_cfg + from federatedscope.core.fed_runner import FedRunner + + init_cfg = global_cfg.clone() + init_cfg.merge_from_file( + "federatedscope/example_configs/single_process.yaml") + init_cfg.merge_from_list(["optimizer.lr", float(x[0])]) + + setup_logger(init_cfg) + setup_seed(init_cfg.seed) + logging.info("======> try {}".format(x)) + + # federated dataset might change the number of clients + # thus, we allow the creation procedure of dataset to modify the global cfg object + data, modified_cfg = get_data(config=init_cfg.clone()) + init_cfg.merge_from_other_cfg(modified_cfg) + + init_cfg.freeze() + + runner = FedRunner(data=data, + server_class=get_server_cls(init_cfg), + client_class=get_client_cls(init_cfg), + config=init_cfg.clone()) + results = runner.run() + + # so that we could modify cfg in the next trial + init_cfg.defrost() + + return [results['client_summarized_weighted_avg']['test_avg_loss']] + + +def our_target_func(x): + return np.asarray([eval_fl_algo(elem) for elem in x]) + + +def main(): + #target_function, space = forrester_function() + target_function = our_target_func + space = ParameterSpace([ContinuousParameter('lr', 1e-4, .75)]) + x_plot = np.linspace(space.parameters[0].min, space.parameters[0].max, + 200)[:, None] + #y_plot = target_function(x_plot) + X_init = np.array([[0.005], [0.05], [0.5]]) + Y_init = target_function(X_init) + + bo = GPBayesianOptimization(variables_list=space.parameters, + X=X_init, + Y=Y_init) + bo.run_optimization(target_function, 15) + + mu_plot, var_plot = bo.model.predict(x_plot) + + plt.figure(figsize=(12, 8)) + plt.plot(bo.loop_state.X, + bo.loop_state.Y, + "ro", + markersize=10, + label="Observations") + #plt.plot(x_plot, y_plot, "k", label="Objective Function") + #plt.plot(x_plot, mu_plot, "C0", label="Model") + plt.fill_between(x_plot[:, 0], + mu_plot[:, 0] + np.sqrt(var_plot)[:, 0], + mu_plot[:, 0] - np.sqrt(var_plot)[:, 0], + color="C0", + alpha=0.6) + + plt.fill_between(x_plot[:, 0], + mu_plot[:, 0] + 2 * np.sqrt(var_plot)[:, 0], + mu_plot[:, 0] - 2 * np.sqrt(var_plot)[:, 0], + color="C0", + alpha=0.4) + + plt.fill_between(x_plot[:, 0], + mu_plot[:, 0] + 3 * np.sqrt(var_plot)[:, 0], + mu_plot[:, 0] - 3 * np.sqrt(var_plot)[:, 0], + color="C0", + alpha=0.2) + plt.legend(loc=2, prop={'size': LEGEND_SIZE}) + plt.xlabel(r"$x$") + plt.ylabel(r"$f(x)$") + plt.grid(True) + plt.xlim(0, 0.75) + + #plt.show() + plt.savefig("bbo.pdf", bbox_inches='tight') + plt.close() + + +if __name__ == "__main__": + main() diff --git a/enviroment/requirements-torch1.8-application.txt b/enviroment/requirements-torch1.8-application.txt new file mode 100644 index 000000000..6f52a6825 --- /dev/null +++ b/enviroment/requirements-torch1.8-application.txt @@ -0,0 +1,23 @@ +numpy==1.19.5 +scikit-learn==1.0 +scipy==1.6.0 +pandas==1.2.1 +scikit-learn +pytorch==1.8.0 +torchvision==0.9.0 +torchaudio==0.8.0 +cudatoolkit==10.2.89 +wandb +tensorboard +tensorboardX +grpcio +grpcio-tools +protobuf==3.19.1 +setuptools==58.0.4 +pyg==2.0.1 +rdkit=2021.09.4 +sentencepiece +textgrid +typeguard + + diff --git a/federatedscope/main.py b/federatedscope/main.py index 2c70f2469..b59d98edd 100644 --- a/federatedscope/main.py +++ b/federatedscope/main.py @@ -1,7 +1,7 @@ import os import sys -DEV_MODE = True # simplify the federatedscope re-setup everytime we change the source codes of federatedscope +DEV_MODE = False # simplify the federatedscope re-setup everytime we change the source codes of federatedscope if DEV_MODE: file_dir = os.path.join(os.path.dirname(__file__), '..') sys.path.append(file_dir) @@ -19,23 +19,23 @@ del os.environ['http_proxy'] if __name__ == '__main__': - + init_cfg = global_cfg.clone() args = parse_args() - global_cfg.merge_from_file(args.cfg_file) - global_cfg.merge_from_list(args.opts) + init_cfg.merge_from_file(args.cfg_file) + init_cfg.merge_from_list(args.opts) - setup_logger(global_cfg) - setup_seed(global_cfg.seed) + setup_logger(init_cfg) + setup_seed(init_cfg.seed) # federated dataset might change the number of clients # thus, we allow the creation procedure of dataset to modify the global cfg object - data, modified_cfg = get_data(config=global_cfg.clone()) - global_cfg.merge_from_other_cfg(modified_cfg) + data, modified_cfg = get_data(config=init_cfg.clone()) + init_cfg.merge_from_other_cfg(modified_cfg) - global_cfg.freeze() + init_cfg.freeze() runner = FedRunner(data=data, - server_class=get_server_cls(global_cfg), - client_class=get_client_cls(global_cfg), - config=global_cfg.clone()) + server_class=get_server_cls(init_cfg), + client_class=get_client_cls(init_cfg), + config=init_cfg.clone()) _ = runner.run()