Add xgb HE eval (#549)

federatedscope/core/configs/cfg_fl_setting.py

@@ -96,6 +96,7 @@ def extend_fl_setting_cfg(cfg):
     # Default values for 'dp': {'bucket_num':100, 'epsilon':None}
     # Default values for 'op_boost': {'algo':'global', 'lower_bound':1,
     #                                 'upper_bound':100, 'epsilon':2}
+    cfg.vertical.eval_protection = ''  # ['', 'he']
     cfg.vertical.data_size_for_debug = 0  # use a subset for debug in vfl,
     # 0 indicates using the entire dataset (disable debug mode)
 

federatedscope/vertical_fl/loss/binary_cls.py

@@ -1,4 +1,5 @@
 import numpy as np
+from sklearn import metrics
 
 
 class BinaryClsLoss(object):
@@ -27,9 +28,10 @@ def _process_y_pred(self, y_pred):
 
     def get_metric(self, y, y_pred):
         y_pred = self._process_y_pred(y_pred)
+        auc = metrics.roc_auc_score(y, y_pred)
         y_pred = (y_pred >= 0.5).astype(np.float32)
         acc = np.sum(y_pred == y) / len(y)
-        return {'acc': acc}
+        return {'acc': acc, 'auc': auc}
 
     def get_loss(self, y, y_pred):
         y_pred = self._process_y_pred(y_pred)

federatedscope/vertical_fl/utils.py

@@ -1,6 +1,8 @@
 from federatedscope.vertical_fl.xgb_base.worker import wrap_client_for_train, \
     wrap_server_for_train, wrap_client_for_evaluation, \
     wrap_server_for_evaluation
+from federatedscope.vertical_fl.xgb_base.worker.he_evaluation_wrapper\
+    import wrap_client_for_he_evaluation
 
 
 def wrap_vertical_server(server, config):
@@ -13,7 +15,9 @@ def wrap_vertical_server(server, config):
 
 def wrap_vertical_client(client, config):
     if config.vertical.algo in ['xgb', 'gbdt', 'rf']:
+        if config.vertical.eval_protection == 'he':
+            client = wrap_client_for_he_evaluation(client)
+        else:
+            client = wrap_client_for_evaluation(client)
         client = wrap_client_for_train(client)
-        client = wrap_client_for_evaluation(client)
-
     return client

federatedscope/vertical_fl/xgb_base/baseline/gbdt_base_on_abalone.yaml

@@ -0,0 +1,36 @@
+use_gpu: False
+device: 0
+backend: torch
+federate:
+  mode: standalone
+  client_num: 2
+model:
+  type: gbdt_tree
+  lambda_: 0.1
+  gamma: 0
+  num_of_trees: 10
+  max_tree_depth: 4
+data:
+  root: data/
+  type: abalone
+  splits: [0.8, 0.2]
+dataloader:
+  type: raw
+  batch_size: 2000
+criterion:
+  type: RegressionMSELoss
+trainer:
+  type: verticaltrainer
+train:
+  optimizer:
+    # learning rate for xgb model
+    eta: 0.5
+vertical:
+  use: True
+  dims: [4, 8]
+  algo: 'gbdt'
+  data_size_for_debug: 2000
+  feature_subsample_ratio: 0.5
+eval:
+  freq: 3
+  best_res_update_round_wise_key: test_loss

federatedscope/vertical_fl/xgb_base/baseline/rf_base_on_abalone.yaml

@@ -0,0 +1,32 @@
+use_gpu: False
+device: 0
+backend: torch
+federate:
+  mode: standalone
+  client_num: 2
+model:
+  type: random_forest
+  lambda_: 0.1
+  gamma: 0
+  num_of_trees: 10
+  max_tree_depth: 4
+data:
+  root: data/
+  type: abalone
+  splits: [0.8, 0.2]
+dataloader:
+  type: raw
+  batch_size: 2000
+criterion:
+  type: RegressionMSELoss
+trainer:
+  type: verticaltrainer
+vertical:
+  use: True
+  dims: [4, 8]
+  algo: 'rf'
+  data_size_for_debug: 1500
+  feature_subsample_ratio: 0.5
+eval:
+  freq: 3
+  best_res_update_round_wise_key: test_loss

federatedscope/vertical_fl/xgb_base/baseline/xgb_base_on_abalone.yaml

@@ -16,7 +16,7 @@ data:
   splits: [0.8, 0.2]
 dataloader:
   type: raw
-  batch_size: 4000
+  batch_size: 2000
 criterion:
   type: RegressionMSELoss
 trainer:

federatedscope/vertical_fl/xgb_base/baseline/xgb_base_on_adult_by_he_eval.yaml

@@ -0,0 +1,36 @@
+use_gpu: False
+device: 0
+backend: torch
+federate:
+  mode: standalone
+  client_num: 2
+model:
+  type: xgb_tree
+  lambda_: 0.1
+  gamma: 0
+  num_of_trees: 10
+  max_tree_depth: 3
+data:
+  root: data/
+  type: adult
+  splits: [1.0, 0.0]
+dataloader:
+  type: raw
+  batch_size: 2000
+criterion:
+  type: CrossEntropyLoss
+trainer:
+  type: verticaltrainer
+train:
+  optimizer:
+    # learning rate for xgb model
+    eta: 0.5
+vertical:
+  use: True
+  dims: [7, 14]
+  algo: 'xgb'
+  eval_protection: 'he'
+  data_size_for_debug: 2000
+eval:
+  freq: 3
+  best_res_update_round_wise_key: test_loss

federatedscope/vertical_fl/xgb_base/baseline/xgb_feature_order_dp_on_abalone.yaml

@@ -0,0 +1,38 @@
+use_gpu: False
+device: 0
+backend: torch
+federate:
+  mode: standalone
+  client_num: 2
+model:
+  type: xgb_tree
+  lambda_: 0.1
+  gamma: 0
+  num_of_trees: 10
+  max_tree_depth: 3
+data:
+  root: data/
+  type: abalone
+  splits: [0.8, 0.2]
+dataloader:
+  type: raw
+  batch_size: 2000
+criterion:
+  type: RegressionMSELoss
+trainer:
+  type: verticaltrainer
+train:
+  optimizer:
+    # learning rate for xgb model
+    eta: 0.5
+vertical:
+  use: True
+  dims: [4, 8]
+  algo: 'xgb'
+  protect_object: 'feature_order'
+  protect_method: 'dp'
+  protect_args: [ { 'bucket_num': 100, 'epsilon': 10 } ]
+  data_size_for_debug: 2000
+eval:
+  freq: 5
+  best_res_update_round_wise_key: test_loss

federatedscope/vertical_fl/xgb_base/worker/XGBClient.py

@@ -4,6 +4,8 @@
 
 from federatedscope.core.workers import Client
 from federatedscope.core.message import Message
+from federatedscope.vertical_fl.Paillier import \
+            abstract_paillier
 
 logger = logging.getLogger(__name__)
 
@@ -30,6 +32,11 @@ def __init__(self,
         self.msg_buffer = {'train': {}, 'eval': {}}
         self.client_num = self._cfg.federate.client_num
 
+        if self._cfg.vertical.eval_protection == 'he':
+            keys = abstract_paillier.generate_paillier_keypair(
+                n_length=self._cfg.vertical.key_size)
+            self.public_key, self.private_key = keys
+
         self.feature_order = None
         self.merged_feature_order = None
 

federatedscope/vertical_fl/xgb_base/worker/__init__.py

@@ -4,8 +4,11 @@
     wrap_server_for_train, wrap_client_for_train
 from federatedscope.vertical_fl.xgb_base.worker.evaluation_wrapper import \
     wrap_server_for_evaluation, wrap_client_for_evaluation
+from federatedscope.vertical_fl.xgb_base.worker.he_evaluation_wrapper\
+    import wrap_client_for_he_evaluation
 
 __all__ = [
     'XGBServer', 'XGBClient', 'wrap_server_for_train', 'wrap_client_for_train',
-    'wrap_server_for_evaluation', 'wrap_client_for_evaluation'
+    'wrap_server_for_evaluation', 'wrap_client_for_evaluation',
+    'wrap_client_for_he_evaluation'
 ]