Merge pull request #212 from sgunasekar/patch-1

Ignore errors in loading tokenizer from_cache

archai/datasets/nlp/fast_hf_dataset_provider.py

@@ -406,12 +406,12 @@ class FastHfDatasetProvider(DatasetProvider):
         cache_test_file = cache_dir / "test.npy"
 
         tokenizer_file = cache_dir / "tokenizer.pkl"
-        if not tokenizer_file.is_file():
-            logger.warn(f"Could not find tokenizer in {cache_dir}.")
-            tokenizer = None
-        else:
+        try:
             with open(tokenizer_file, "rb") as f:
                 tokenizer = pickle.load(f)
+        except:
+            logger.warn(f"Could not load tokenizer.pkl from {cache_dir}.")
+            tokenizer = None
 
         return FastHfDatasetProvider(cache_train_file, cache_validation_file, cache_test_file, tokenizer=tokenizer)