From cc2366bbb9d7a55fb158d109a45fa20f8642bbb4 Mon Sep 17 00:00:00 2001 From: Masatoshi Suzuki Date: Mon, 29 Mar 2021 23:26:15 +0900 Subject: [PATCH] Ignore not initialized NO_CONFIG_TOKENIZERs (#10936) --- src/transformers/models/auto/tokenization_auto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 346e62645..bf58b7509 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -288,7 +288,7 @@ def tokenizer_class_from_name(class_name: str): all_tokenizer_classes = ( [v[0] for v in TOKENIZER_MAPPING.values() if v[0] is not None] + [v[1] for v in TOKENIZER_MAPPING.values() if v[1] is not None] - + NO_CONFIG_TOKENIZER + + [v for v in NO_CONFIG_TOKENIZER if v is not None] ) for c in all_tokenizer_classes: if c.__name__ == class_name: