This commit is contained in:
nonstoptimm 2021-09-23 12:57:18 +02:00
Родитель dabf223b32
Коммит 20fb5ba240
2 изменённых файлов: 21 добавлений и 21 удалений

Просмотреть файл

@ -11,6 +11,7 @@ import configparser
from pathlib import Path
import pandas as pd
import json
import spacy
try:
from azure.common.credentials import ServicePrincipalCredentials
@ -274,6 +275,26 @@ flair_model_file_lookup = {
'xx' : 'ner-multi-fast.pt'
}
spacy_model_lookup = {
'en':'en_core_web_sm',
'de':'de_core_news_sm',
'fr':'fr_core_news_sm',
'es':'es_core_news_sm',
'it':'it_core_news_sm',
'xx':'xx_ent_wiki_sm'
}
def load_spacy_model(language='xx', disable=[]):
"""Load spacy models depending on language"""
try:
nlp = spacy.load(spacy_model_lookup[language], disable=disable)
except OSError:
logging.warning(f'[INFO] Downloading spacy language model for {language}')
from spacy.cli import download
download(spacy_model_lookup[language])
nlp = spacy.load(spacy_model_lookup[language], disable=disable)
return nlp
def get_flair_model(language, object_type):
if object_type == 'model':
lookup = flair_model_lookup

Просмотреть файл

@ -30,27 +30,6 @@ import custom as cu
import data as dt
import helper as he
spacy_model_lookup = {
'en':'en_core_web_sm',
'de':'de_core_news_sm',
'fr':'fr_core_news_sm',
'es':'es_core_news_sm',
'it':'it_core_news_sm',
'xx':'xx_ent_wiki_sm'
}
def load_spacy_model(language='xx', disable=[]):
"""Load spacy models depending on language"""
try:
nlp = spacy.load(spacy_model_lookup[language], disable=disable)
except OSError:
logging.warning(f'[INFO] Downloading spacy language model for {language}')
from spacy.cli import download
download(spacy_model_lookup[language])
nlp = spacy.load(spacy_model_lookup[language], disable=disable)
return nlp
def load_flair_model(path=None, language='xx', task='ner'):
"""Load flair models depending on language"""
if task == 'ner':