diff --git a/archai/common/apex_utils.py b/archai/common/apex_utils.py
index 59a2f158..fa82aaf3 100644
--- a/archai/common/apex_utils.py
+++ b/archai/common/apex_utils.py
@@ -152,7 +152,7 @@ class ApexUtils:
     def is_dist(self)->bool:
         return self._enabled and self._distributed_enabled
     def is_master(self)->bool:
-        return self.global_rank == 0
+        return self.global_rank == 0 and self.local_rank == 0
 
     def _log_info(self, d:dict)->None:
         if self.logger is not None: