upgrade trust

Author: FerTV

nebula/addons/trustworthiness/factsheet.py

@@ -67,7 +67,7 @@ def populate_factsheet_pre_train(self, data, scenario_name):
                         poisoned_node_percent = 0
                         poisoned_sample_percent = 0
                         poisoned_noise_percent = 0
-                    with_reputation = data["with_reputation"]
+                    with_reputation = data["reputation"]["enabled"]
                     is_dynamic_topology = False # data["is_dynamic_topology"]
                     is_dynamic_aggregation = False # data["is_dynamic_aggregation"]
                     target_aggregation = False # data["target_aggregation"]
@@ -125,15 +125,14 @@ def populate_factsheet_pre_train(self, data, scenario_name):
                     factsheet["configuration"]["trainable_param_num"] = model.count_parameters()
                     factsheet["configuration"]["local_update_steps"] = 1
 
+                    f.seek(0)
+                    f.truncate()
+                    json.dump(factsheet, f, indent=4)
+
             except JSONDecodeError as e:
                 logging.warning(f"{factsheet_file} is invalid")
                 logging.error(e)
 
-            f.seek(0)
-            f.truncate()
-            json.dump(factsheet, f, indent=4)
-            f.close()
-
     def populate_factsheet_post_train(self, scenario_name, start_time, end_time):
         """
         Populates the factsheet with values after the training.
@@ -198,14 +197,16 @@ def populate_factsheet_post_train(self, scenario_name, start_time, end_time):
 
                 factsheet["fairness"]["selection_cv"] = 1
 
-                count_class_samples(scenario_name, dataloaders_files)
+                # count_class_samples(scenario_name, dataloaders_files)
+                
+                ## FER
 
-                with open(f"{files_dir}/count_class.json", "r") as file:
-                    class_distribution = json.load(file)
+                # with open(f"{files_dir}/count_class.json", "r") as file:
+                #     class_distribution = json.load(file)
 
-                class_samples_sizes = [x for x in class_distribution.values()]
-                class_imbalance = get_cv(list=class_samples_sizes)
-                factsheet["fairness"]["class_imbalance"] = 1 if class_imbalance > 1 else class_imbalance
+                # class_samples_sizes = [x for x in class_distribution.values()]
+                # class_imbalance = get_cv(list=class_samples_sizes)
+                # factsheet["fairness"]["class_imbalance"] = 1 if class_imbalance > 1 else class_imbalance
 
                 with open(train_model_file, "rb") as file:
                     lightning_model = pickle.load(file)
@@ -273,11 +274,10 @@ def populate_factsheet_post_train(self, scenario_name, start_time, end_time):
                     factsheet["sustainability"]["emissions_communication_uplink"] = check_field_filled(factsheet, ["sustainability", "emissions_communication_uplink"], factsheet["system"]["total_upload_bytes"] * 2.24e-10 * factsheet["sustainability"]["avg_carbon_intensity_clients"], "")
                     factsheet["sustainability"]["emissions_communication_downlink"] = check_field_filled(factsheet, ["sustainability", "emissions_communication_downlink"], factsheet["system"]["total_download_bytes"] * 2.24e-10 * factsheet["sustainability"]["avg_carbon_intensity_server"], "")
 
-            except JSONDecodeError as e:
-                logging.warning(f"{factsheet_file} is invalid")
-                logging.error(e)
+                f.seek(0)
+                f.truncate()
+                json.dump(factsheet, f, indent=4)
 
-            f.seek(0)
-            f.truncate()
-            json.dump(factsheet, f, indent=4)
-            f.close()
+            except JSONDecodeError as e:
+                logging.info(f"{factsheet_file} is invalid")
+                logging.error(e)

nebula/addons/trustworthiness/trustworthiness.py

@@ -166,40 +166,40 @@ async def _generate_factsheet(self, trust_config, experiment_name):
         factsheet.populate_factsheet_post_train(experiment_name, self._start_time, self._end_time)
         logging.info("[FER] factsheet post train done")
 
-        data_file_path = os.path.join(os.environ.get('NEBULA_CONFIG_DIR'), experiment_name, "scenario.json")
-        with open(data_file_path, 'r') as data_file:
-            data = json.load(data_file)
+        # data_file_path = os.path.join(os.environ.get('NEBULA_CONFIG_DIR'), experiment_name, "scenario.json")
+        # with open(data_file_path, 'r') as data_file:
+        #     data = json.load(data_file)
 
-            weights = {
-                "robustness": float(data["robustness_pillar"]),
-                "resilience_to_attacks": float(data["resilience_to_attacks"]),
-                "algorithm_robustness": float(data["algorithm_robustness"]),
-                "client_reliability": float(data["client_reliability"]),
-                "privacy": float(data["privacy_pillar"]),
-                "technique": float(data["technique"]),
-                "uncertainty": float(data["uncertainty"]),
-                "indistinguishability": float(data["indistinguishability"]),
-                "fairness": float(data["fairness_pillar"]),
-                "selection_fairness": float(data["selection_fairness"]),
-                "performance_fairness": float(data["performance_fairness"]),
-                "class_distribution": float(data["class_distribution"]),
-                "explainability": float(data["explainability_pillar"]),
-                "interpretability": float(data["interpretability"]),
-                "post_hoc_methods": float(data["post_hoc_methods"]),
-                "accountability": float(data["accountability_pillar"]),
-                "factsheet_completeness":  float(data["factsheet_completeness"]),
-                "architectural_soundness": float(data["architectural_soundness_pillar"]),
-                "client_management": float(data["client_management"]),
-                "optimization": float(data["optimization"]),
-                "sustainability": float(data["sustainability_pillar"]),
-                "energy_source": float(data["energy_source"]),
-                "hardware_efficiency": float(data["hardware_efficiency"]),
-                "federation_complexity": float(data["federation_complexity"])
-            }
+        #     weights = {
+        #         "robustness": float(data["robustness_pillar"]),
+        #         "resilience_to_attacks": float(data["resilience_to_attacks"]),
+        #         "algorithm_robustness": float(data["algorithm_robustness"]),
+        #         "client_reliability": float(data["client_reliability"]),
+        #         "privacy": float(data["privacy_pillar"]),
+        #         "technique": float(data["technique"]),
+        #         "uncertainty": float(data["uncertainty"]),
+        #         "indistinguishability": float(data["indistinguishability"]),
+        #         "fairness": float(data["fairness_pillar"]),
+        #         "selection_fairness": float(data["selection_fairness"]),
+        #         "performance_fairness": float(data["performance_fairness"]),
+        #         "class_distribution": float(data["class_distribution"]),
+        #         "explainability": float(data["explainability_pillar"]),
+        #         "interpretability": float(data["interpretability"]),
+        #         "post_hoc_methods": float(data["post_hoc_methods"]),
+        #         "accountability": float(data["accountability_pillar"]),
+        #         "factsheet_completeness":  float(data["factsheet_completeness"]),
+        #         "architectural_soundness": float(data["architectural_soundness_pillar"]),
+        #         "client_management": float(data["client_management"]),
+        #         "optimization": float(data["optimization"]),
+        #         "sustainability": float(data["sustainability_pillar"]),
+        #         "energy_source": float(data["energy_source"]),
+        #         "hardware_efficiency": float(data["hardware_efficiency"]),
+        #         "federation_complexity": float(data["federation_complexity"])
+        #     }
 
-            trust_metric_manager = TrustMetricManager(self._start_time)
-            trust_metric_manager.evaluate(experiment_name, weights, use_weights=True)
-            logging.info("[FER] evaluation done")
+        #     trust_metric_manager = TrustMetricManager(self._start_time)
+        #     trust_metric_manager.evaluate(experiment_name, weights, use_weights=True)
+        #     logging.info("[FER] evaluation done")
 
     async def _process_test_metrics_event(self, tme: TestMetricsEvent):
         cur_loss, cur_acc = await tme.get_event_data()

nebula/core/datasets/datamodule.py

@@ -19,6 +19,7 @@ def __init__(
         test_set_indices,
         local_test_set,
         local_test_set_indices,
+        samples_per_label,
         batch_size=32,
         num_workers=0,
         val_percent=0.1,
@@ -35,6 +36,7 @@ def __init__(
         self.num_workers = num_workers
         self.val_percent = val_percent
         self.seed = seed
+        self.samples_per_label = samples_per_label
 
         self.model_weight = None
 
@@ -93,7 +95,6 @@ def train_dataloader(self):
                 "Train dataset not initialized. Please call setup('fit') before requesting train_dataloader."
             )
         logging_training.info(f"Train set size: {len(self.data_train)}")
-        logging.info("[FER] train_dataloader")
         return DataLoader(
             self.data_train,
             batch_size=self.batch_size,
@@ -125,7 +126,6 @@ def test_dataloader(self):
             )
         logging_training.info(f"Local test set size: {len(self.local_te_subset)}")
         logging_training.info(f"Global test set size: {len(self.global_te_subset)}")
-        logging.info("[FER] test_dataloader")
         return [
             DataLoader(
                 self.local_te_subset,

nebula/core/node.py

@@ -14,6 +14,7 @@
 
 sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
 import logging
+from collections import Counter
 
 from nebula.config.config import Config
 from nebula.core.datasets.cifar10.cifar10 import CIFAR10PartitionHandler
@@ -149,6 +150,7 @@ async def main(config):
     dataset = NebulaPartition(handler=handler, config=config)
     dataset.load_partition()
     dataset.log_partition()
+    samples_per_label = Counter(dataset.get_train_labels())
 
     datamodule = DataModule(
         train_set=dataset.train_set,
@@ -159,6 +161,7 @@ async def main(config):
         local_test_set_indices=dataset.local_test_indices,
         num_workers=num_workers,
         batch_size=batch_size,
+        samples_per_label = samples_per_label
     )
 
     trainer = None

nebula/utils.py

@@ -105,6 +105,26 @@ def create_docker_network(cls, network_name, subnet=None, prefix=24):
         finally:
             client.close()  # Ensure the Docker client is closed
 
+    @classmethod
+    def check_docker_by_prefix(cls, prefix):
+        try:
+            # Connect to Docker client
+            client = docker.from_env()
+
+            containers = client.containers.list(all=True)  # `all=True` to include stopped containers
+
+            # Iterate through containers and remove those with the matching prefix
+            for container in containers:
+                if container.name.startswith(prefix):
+                    return True
+                
+            return False
+
+        except docker.errors.APIError:
+            logging.exception("Error interacting with Docker")
+        except Exception:
+            logging.exception("Unexpected error")
+
     @classmethod
     def remove_docker_network(cls, network_name):
         try: