- implement a new mode_based_filtering decorator in Assigner class

- update all the sub-classes that use task_groups to use the decorator
- update fedeval sample workspace to use default assigner, tasks and aggregator
- use of federated-evaluation/aggregator.yaml for FedEval specific workspace example to use round_number as 1
- removed assigner and tasks yaml from defaults/federated-evaluation, superseded by default assigner/tasks
- Rebase 5-Jan-2025.1
- Addressing review comments
Signed-off-by: Shailesh Pant <[email protected]>

openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml

@@ -32,10 +32,12 @@ network :
   defaults : plan/defaults/network.yaml
 
 assigner :
-  defaults : plan/defaults/federated-evaluation/assigner.yaml
-
+  defaults : plan/defaults/assigner.yaml
+  settings :
+    mode : evaluate
+
 tasks :
-  defaults : plan/defaults/federated-evaluation/tasks_torch.yaml
+  defaults : plan/defaults/tasks_torch.yaml
 
 compression_pipeline :
   defaults : plan/defaults/compression_pipeline.yaml

openfl-workspace/workspace/plan/defaults/assigner.yaml

@@ -7,3 +7,8 @@ settings :
         - aggregated_model_validation
         - train
         - locally_tuned_model_validation
+    - name       : evaluate
+      percentage : 1.0
+      tasks      :
+        - aggregated_model_validation
+  mode: train_and_validate

openfl/component/__init__.py

@@ -1,6 +1,7 @@
 # Copyright 2020-2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+"""OpenFL Component Module."""
 
 from openfl.component.aggregator.aggregator import Aggregator
 from openfl.component.assigner.assigner import Assigner

openfl/component/assigner/__init__.py

@@ -1,6 +1,7 @@
 # Copyright 2020-2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+"""OpenFL Assigner Module."""
 
 from openfl.component.assigner.assigner import Assigner
 from openfl.component.assigner.random_grouped_assigner import RandomGroupedAssigner

openfl/component/assigner/assigner.py

@@ -4,6 +4,7 @@
 
 """Assigner module."""
 
+from functools import wraps
 
 class Assigner:
     r"""
@@ -35,18 +36,22 @@ class Assigner:
         \* - ``tasks`` argument is taken from ``tasks`` section of FL plan YAML file.
     """
 
-    def __init__(self, tasks, authorized_cols, rounds_to_train, **kwargs):
+    def __init__(
+        self, tasks, authorized_cols, rounds_to_train, mode: str = "train_and_validate", **kwargs
+    ):
         """Initializes the Assigner.
 
         Args:
             tasks (list of object): List of tasks to assign.
             authorized_cols (list of str): Collaborators.
             rounds_to_train (int): Number of training rounds.
+            mode (str, optional): Mode for filtering tasks. Defaults to "train_and_validate".
             **kwargs: Additional keyword arguments.
         """
         self.tasks = tasks
         self.authorized_cols = authorized_cols
         self.rounds = rounds_to_train
+        self.mode = mode
         self.all_tasks_in_groups = []
 
         self.task_group_collaborators = {}
@@ -93,3 +98,31 @@ def get_aggregation_type_for_task(self, task_name):
         if "aggregation_type" not in self.tasks[task_name]:
             return None
         return self.tasks[task_name]["aggregation_type"]
+
+    @classmethod
+    def mode_based_filtering(cls, func):
+        """Decorator to filter task groups based on mode.
+
+        This decorator should be applied to define_task_assignments() method
+        in Assigner subclasses to handle mode-based filtering.
+        """
+        @wraps(func)
+        def wrapper(self, *args, **kwargs):
+            # Filter task groups based on mode before assignment
+            if hasattr(self, "mode"):
+                self.task_groups = [
+                    group for group in self.task_groups 
+                    if group["name"].startswith(self.mode)
+                ]
+
+                if not self.task_groups:
+                    raise ValueError(f"No task groups found for mode: {self.mode}")
+
+                # Mode-specific validations
+                if self.mode == "evaluate":
+                    assert self.rounds == 1, "Number of rounds should be 1 for evaluate mode"
+
+            # Call the original method
+            return func(self, *args, **kwargs)
+
+        return wrapper

openfl/component/assigner/random_grouped_assigner.py

@@ -38,11 +38,12 @@ def __init__(self, task_groups, **kwargs):
 
         Args:
             task_groups (list of object): Task groups to assign.
-            **kwargs: Additional keyword arguments.
+            **kwargs: Additional keyword arguments, including mode.
         """
         self.task_groups = task_groups
         super().__init__(**kwargs)
 
+    @Assigner.mode_based_filtering
     def define_task_assignments(self):
         """Define task assignments for each round and collaborator.
 

openfl/component/assigner/static_grouped_assigner.py

@@ -42,6 +42,7 @@ def __init__(self, task_groups, **kwargs):
         self.task_groups = task_groups
         super().__init__(**kwargs)
 
+    @Assigner.mode_based_filtering
     def define_task_assignments(self):
         """Define task assignments for each round and collaborator.