Ignore not initialized NO_CONFIG_TOKENIZERs (#10936)
This commit is contained in:
Родитель
ddea8771c6
Коммит
cc2366bbb9
|
@ -288,7 +288,7 @@ def tokenizer_class_from_name(class_name: str):
|
|||
all_tokenizer_classes = (
|
||||
[v[0] for v in TOKENIZER_MAPPING.values() if v[0] is not None]
|
||||
+ [v[1] for v in TOKENIZER_MAPPING.values() if v[1] is not None]
|
||||
+ NO_CONFIG_TOKENIZER
|
||||
+ [v for v in NO_CONFIG_TOKENIZER if v is not None]
|
||||
)
|
||||
for c in all_tokenizer_classes:
|
||||
if c.__name__ == class_name:
|
||||
|
|
Загрузка…
Ссылка в новой задаче