diff --git a/benchmark/FedHPOB/README.md b/benchmark/FedHPOB/README.md index 0d2522794..ee61004ac 100644 --- a/benchmark/FedHPOB/README.md +++ b/benchmark/FedHPOB/README.md @@ -8,7 +8,14 @@ We highly recommend running FedHPO-B with conda. ### Step 0. Dependency -* FedHPO-B is built on [FederatedScope](https://github.com/alibaba/FederatedScope), please see [Installation](https://github.com/alibaba/FederatedScope#step-1-installation) for install FederatedScope. +* FedHPO-B is built on a stable [FederatedScope](https://github.com/alibaba/FederatedScope), please see [Installation](https://github.com/alibaba/FederatedScope#step-1-installation) for install FederatedScope. + + ```bash + git clone https://github.com/alibaba/FederatedScope.git + git checkout a653102c6b8d5421d2874077594df0b2e29401a1 + cd FederatedScope + pip install -e . + ``` * (Optianal) In order to reproduce the results in our paper, please consider installing the following packages via: @@ -55,21 +62,25 @@ Fortunately, we provide tools to automatically convert from tabular data to surr ### Step3. Start running ```python +from fedhpob.config import fhb_cfg from fedhpob.benchmarks import TabularBenchmark -# benchmark = TabularBenchmark('cnn', 'femnist', 'avg') # get hyperparameters space -config_space = benchmark.get_configuration_space() +config_space = benchmark.get_configuration_space(CS=True) # get fidelity space -fidelity_space = benchmark.get_fidelity_space() +fidelity_space = benchmark.get_fidelity_space(CS=True) # get results res = benchmark(config_space.sample_configuration(), fidelity_space.sample_configuration(), + fhb_cfg=fhb_cfg, seed=12345) + +print(res) + ``` ## Reproduce the results in our paper @@ -152,5 +163,4 @@ Available tabular triplets look-up table: | mlp | 146821@openml | avg | | mlp | 146821@openml | opt | | mlp | 146822@openml | avg | -| mlp | 146822@openml | opt | - +| mlp | 146822@openml | opt | \ No newline at end of file diff --git a/benchmark/FedHPOB/fedhpob/benchmarks/tabular_benchmark.py b/benchmark/FedHPOB/fedhpob/benchmarks/tabular_benchmark.py index b773c4412..d3e4e4b87 100644 --- a/benchmark/FedHPOB/fedhpob/benchmarks/tabular_benchmark.py +++ b/benchmark/FedHPOB/fedhpob/benchmarks/tabular_benchmark.py @@ -79,11 +79,21 @@ def objective_function(self, return {'function_value': function_value, 'cost': cost} - def get_configuration_space(self): - return dict2cfg(self.meta_info['configuration_space']) + def get_configuration_space(self, CS=False): + if not CS: + return self.meta_info['configuration_space'] + tmp_dict = {} + for key in self.meta_info['configuration_space']: + tmp_dict[key] = list(self.meta_info['configuration_space'][key]) + return dict2cfg(tmp_dict) - def get_fidelity_space(self): - return dict2cfg(self.meta_info['fidelity_space']) + def get_fidelity_space(self, CS=False): + if not CS: + return self.meta_info['fidelity_space'] + tmp_dict = {} + for key in self.meta_info['fidelity_space']: + tmp_dict[key] = list(self.meta_info['fidelity_space'][key]) + return dict2cfg(tmp_dict) def get_meta_info(self): return {