зеркало из https://github.com/microsoft/archai.git
Merge pull request #212 from sgunasekar/patch-1
Ignore errors in loading tokenizer from_cache
This commit is contained in:
Коммит
ec72afd749
|
@ -406,12 +406,12 @@ class FastHfDatasetProvider(DatasetProvider):
|
|||
cache_test_file = cache_dir / "test.npy"
|
||||
|
||||
tokenizer_file = cache_dir / "tokenizer.pkl"
|
||||
if not tokenizer_file.is_file():
|
||||
logger.warn(f"Could not find tokenizer in {cache_dir}.")
|
||||
tokenizer = None
|
||||
else:
|
||||
try:
|
||||
with open(tokenizer_file, "rb") as f:
|
||||
tokenizer = pickle.load(f)
|
||||
except:
|
||||
logger.warn(f"Could not load tokenizer.pkl from {cache_dir}.")
|
||||
tokenizer = None
|
||||
|
||||
return FastHfDatasetProvider(cache_train_file, cache_validation_file, cache_test_file, tokenizer=tokenizer)
|
||||
|
||||
|
|
Загрузка…
Ссылка в новой задаче