зеркало из https://github.com/microsoft/presidio.git
Updates to deny_list recognizer (#858)
This commit is contained in:
Omri Mendels
GitHub
Родитель
68b084cf3b
Коммит
8c661fef34
|
|
@ -445,13 +445,13 @@ def test_given_ad_hoc_deny_list_recognizer_the_right_entities_are_returned():
|
|||
{
|
||||
"name": "Mr. Recognizer",
|
||||
"supported_language": "en",
|
||||
"deny_list": ["Mr", "Mr.", "Mister"],
|
||||
"deny_list": ["Mr.", "Mister"],
|
||||
"supported_entity":"MR_TITLE"
|
||||
},
|
||||
{
|
||||
"name": "Ms. Recognizer",
|
||||
"supported_language": "en",
|
||||
"deny_list": ["Ms", "Ms.", "Miss", "Mrs", "Mrs."],
|
||||
"deny_list": ["Ms.", "Miss", "Mrs."],
|
||||
"supported_entity":"MS_TITLE"
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ class NlpArtifacts:
|
|||
return_dict = self.__dict__.copy()
|
||||
|
||||
# Ignore NLP engine as it's not serializable currently
|
||||
del return_dict['nlp_engine']
|
||||
del return_dict["nlp_engine"]
|
||||
|
||||
# Converting spaCy tokens and spans to string as they are not serializable
|
||||
if "tokens" in return_dict:
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ class PatternRecognizer(LocalRecognizer):
|
|||
:param deny_list: A list of words to detect,
|
||||
in case our recognizer uses a predefined list of words (deny list)
|
||||
:param context: list of context words
|
||||
:param deny_list_score: confidence score for a term
|
||||
identified using a deny-list
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
|
@ -34,6 +36,7 @@ class PatternRecognizer(LocalRecognizer):
|
|||
patterns: List[Pattern] = None,
|
||||
deny_list: List[str] = None,
|
||||
context: List[str] = None,
|
||||
deny_list_score: float = 1.0,
|
||||
version: str = "0.0.1",
|
||||
):
|
||||
|
||||
|
|
@ -57,9 +60,10 @@ class PatternRecognizer(LocalRecognizer):
|
|||
else:
|
||||
self.patterns = patterns
|
||||
self.context = context
|
||||
self.deny_list_score = deny_list_score
|
||||
|
||||
if deny_list:
|
||||
deny_list_pattern = self.__deny_list_to_regex(deny_list)
|
||||
deny_list_pattern = self._deny_list_to_regex(deny_list)
|
||||
self.patterns.append(deny_list_pattern)
|
||||
self.deny_list = deny_list
|
||||
else:
|
||||
|
|
@ -92,18 +96,20 @@ class PatternRecognizer(LocalRecognizer):
|
|||
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
def __deny_list_to_regex(deny_list: List[str]) -> Pattern:
|
||||
def _deny_list_to_regex(self, deny_list: List[str]) -> Pattern:
|
||||
"""
|
||||
Convert a list of word to a matching regex.
|
||||
Convert a list of words to a matching regex.
|
||||
|
||||
To be analyzed by the regex engine as a part of the analyze logic.
|
||||
To be analyzed by the analyze method as any other regex patterns.
|
||||
|
||||
:param deny_list: the list of words to detect
|
||||
:return:the regex of the words for detection
|
||||
"""
|
||||
regex = r"(?:^|(?<= ))(" + "|".join(deny_list) + r")(?:(?= )|$)"
|
||||
return Pattern(name="deny_list", regex=regex, score=1.0)
|
||||
|
||||
# Escape deny list elements as preparation for regex
|
||||
escaped_deny_list = [re.escape(element) for element in deny_list]
|
||||
regex = r"(?:^|(?<=\W))(" + "|".join(escaped_deny_list) + r")(?:(?=\W)|$)"
|
||||
return Pattern(name="deny_list", regex=regex, score=self.deny_list_score)
|
||||
|
||||
def validate_result(self, pattern_text: str) -> Optional[bool]:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -9,165 +9,165 @@ An IBAN checker is available here: https://www.iban.com/iban-checker
|
|||
"""
|
||||
|
||||
# IBAN parts format
|
||||
CC = u"[A-Z]{2}" # country code
|
||||
CK = u"[0-9]{2}[ ]?" # checksum
|
||||
BOS = u"^"
|
||||
EOS = u"$" # end of string
|
||||
CC = "[A-Z]{2}" # country code
|
||||
CK = "[0-9]{2}[ ]?" # checksum
|
||||
BOS = "^"
|
||||
EOS = "$" # end of string
|
||||
|
||||
A = u"[A-Z][ ]?"
|
||||
A2 = u"([A-Z][ ]?){2}"
|
||||
A3 = u"([A-Z][ ]?){3}"
|
||||
A4 = u"([A-Z][ ]?){4}"
|
||||
A = "[A-Z][ ]?"
|
||||
A2 = "([A-Z][ ]?){2}"
|
||||
A3 = "([A-Z][ ]?){3}"
|
||||
A4 = "([A-Z][ ]?){4}"
|
||||
|
||||
C = u"[a-zA-Z0-9][ ]?"
|
||||
C2 = u"([a-zA-Z0-9][ ]?){2}"
|
||||
C3 = u"([a-zA-Z0-9][ ]?){3}"
|
||||
C4 = u"([a-zA-Z0-9][ ]?){4}"
|
||||
C = "[a-zA-Z0-9][ ]?"
|
||||
C2 = "([a-zA-Z0-9][ ]?){2}"
|
||||
C3 = "([a-zA-Z0-9][ ]?){3}"
|
||||
C4 = "([a-zA-Z0-9][ ]?){4}"
|
||||
|
||||
N = u"[0-9][ ]?"
|
||||
N2 = u"([0-9][ ]?){2}"
|
||||
N3 = u"([0-9][ ]?){3}"
|
||||
N4 = u"([0-9][ ]?){4}"
|
||||
N = "[0-9][ ]?"
|
||||
N2 = "([0-9][ ]?){2}"
|
||||
N3 = "([0-9][ ]?){3}"
|
||||
N4 = "([0-9][ ]?){4}"
|
||||
|
||||
regex_per_country = {
|
||||
# Albania (8n, 16c) ALkk bbbs sssx cccc cccc cccc cccc
|
||||
"AL": u"(AL)" + CK + N4 + N4 + C4 + C4 + C4 + C4,
|
||||
"AL": "(AL)" + CK + N4 + N4 + C4 + C4 + C4 + C4,
|
||||
# Andorra (8n, 12c) ADkk bbbb ssss cccc cccc cccc
|
||||
"AD": u"(AD)" + CK + N4 + N4 + C4 + C4 + C4,
|
||||
"AD": "(AD)" + CK + N4 + N4 + C4 + C4 + C4,
|
||||
# Austria (16n) ATkk bbbb bccc cccc cccc
|
||||
"AT": u"(AT)" + CK + N4 + N4 + N4 + N4,
|
||||
"AT": "(AT)" + CK + N4 + N4 + N4 + N4,
|
||||
# Azerbaijan (4c,20n) AZkk bbbb cccc cccc cccc cccc cccc
|
||||
"AZ": u"(AZ)" + CK + C4 + N4 + N4 + N4 + N4 + N4,
|
||||
"AZ": "(AZ)" + CK + C4 + N4 + N4 + N4 + N4 + N4,
|
||||
# Bahrain (4a,14c) BHkk bbbb cccc cccc cccc cc
|
||||
"BH": u"(BH)" + CK + A4 + C4 + C4 + C4 + C2,
|
||||
"BH": "(BH)" + CK + A4 + C4 + C4 + C4 + C2,
|
||||
# Belarus (4c, 4n, 16c) BYkk bbbb aaaa cccc cccc cccc cccc
|
||||
"BY": u"(BY)" + CK + C4 + N4 + C4 + C4 + C4 + C4,
|
||||
"BY": "(BY)" + CK + C4 + N4 + C4 + C4 + C4 + C4,
|
||||
# Belgium (12n) BEkk bbbc cccc ccxx
|
||||
"BE": u"(BE)" + CK + N4 + N4 + N4,
|
||||
"BE": "(BE)" + CK + N4 + N4 + N4,
|
||||
# Bosnia and Herzegovina (16n) BAkk bbbs sscc cccc ccxx
|
||||
"BA": u"(BA)" + CK + N4 + N4 + N4 + N4,
|
||||
"BA": "(BA)" + CK + N4 + N4 + N4 + N4,
|
||||
# Brazil (23n,1a,1c) BRkk bbbb bbbb ssss sccc cccc ccct n
|
||||
"BR": u"(BR)" + CK + N4 + N4 + N4 + N4 + N4 + N3 + A + C,
|
||||
"BR": "(BR)" + CK + N4 + N4 + N4 + N4 + N4 + N3 + A + C,
|
||||
# Bulgaria (4a,6n,8c) BGkk bbbb ssss ttcc cccc cc
|
||||
"BG": u"(BG)" + CK + A4 + N4 + N + N + C2 + C4 + C2,
|
||||
"BG": "(BG)" + CK + A4 + N4 + N + N + C2 + C4 + C2,
|
||||
# Costa Rica (18n) CRkk 0bbb cccc cccc cccc cc (0 = always zero)
|
||||
"CR": u"(CR)" + CK + u"[0]" + N3 + N4 + N4 + N4 + N2,
|
||||
"CR": "(CR)" + CK + "[0]" + N3 + N4 + N4 + N4 + N2,
|
||||
# Croatia (17n) HRkk bbbb bbbc cccc cccc c
|
||||
"HR": u"(HR)" + CK + N4 + N4 + N4 + N4 + N,
|
||||
"HR": "(HR)" + CK + N4 + N4 + N4 + N4 + N,
|
||||
# Cyprus (8n,16c) CYkk bbbs ssss cccc cccc cccc cccc
|
||||
"CY": u"(CY)" + CK + N4 + N4 + C4 + C4 + C4 + C4,
|
||||
"CY": "(CY)" + CK + N4 + N4 + C4 + C4 + C4 + C4,
|
||||
# Czech Republic (20n) CZkk bbbb ssss sscc cccc cccc
|
||||
"CZ": u"(CZ)" + CK + N4 + N4 + N4 + N4 + N4,
|
||||
"CZ": "(CZ)" + CK + N4 + N4 + N4 + N4 + N4,
|
||||
# Denmark (14n) DKkk bbbb cccc cccc cc
|
||||
"DK": u"(DK)" + CK + N4 + N4 + N4 + N2,
|
||||
"DK": "(DK)" + CK + N4 + N4 + N4 + N2,
|
||||
# Dominican Republic (4a,20n) DOkk bbbb cccc cccc cccc cccc cccc
|
||||
"DO": u"(DO)" + CK + A4 + N4 + N4 + N4 + N4 + N4,
|
||||
"DO": "(DO)" + CK + A4 + N4 + N4 + N4 + N4 + N4,
|
||||
# EAt Timor (19n) TLkk bbbc cccc cccc cccc cxx
|
||||
"TL": u"(TL)" + CK + N4 + N4 + N4 + N4 + N3,
|
||||
"TL": "(TL)" + CK + N4 + N4 + N4 + N4 + N3,
|
||||
# Estonia (16n) EEkk bbss cccc cccc cccx
|
||||
"EE": u"(EE)" + CK + N4 + N4 + N4 + N4,
|
||||
"EE": "(EE)" + CK + N4 + N4 + N4 + N4,
|
||||
# Faroe Islands (14n) FOkk bbbb cccc cccc cx
|
||||
"FO": u"(FO)" + CK + N4 + N4 + N4 + N2,
|
||||
"FO": "(FO)" + CK + N4 + N4 + N4 + N2,
|
||||
# Finland (14n) FIkk bbbb bbcc cccc cx
|
||||
"FI": u"(FI)" + CK + N4 + N4 + N4 + N2,
|
||||
"FI": "(FI)" + CK + N4 + N4 + N4 + N2,
|
||||
# France (10n,11c,2n) FRkk bbbb bsss sscc cccc cccc cxx
|
||||
"FR": u"(FR)" + CK + N4 + N4 + N2 + C2 + C4 + C4 + C + N2,
|
||||
"FR": "(FR)" + CK + N4 + N4 + N2 + C2 + C4 + C4 + C + N2,
|
||||
# Georgia (2c,16n) GEkk bbcc cccc cccc cccc cc
|
||||
"GE": u"(GE)" + CK + C2 + N2 + N4 + N4 + N4 + N2,
|
||||
"GE": "(GE)" + CK + C2 + N2 + N4 + N4 + N4 + N2,
|
||||
# Germany (18n) DEkk bbbb bbbb cccc cccc cc
|
||||
"DE": u"(DE)" + CK + N4 + N4 + N4 + N4 + N2,
|
||||
"DE": "(DE)" + CK + N4 + N4 + N4 + N4 + N2,
|
||||
# Gibraltar (4a,15c) GIkk bbbb cccc cccc cccc ccc
|
||||
"GI": u"(GI)" + CK + A4 + C4 + C4 + C4 + C3,
|
||||
"GI": "(GI)" + CK + A4 + C4 + C4 + C4 + C3,
|
||||
# Greece (7n,16c) GRkk bbbs sssc cccc cccc cccc ccc
|
||||
"GR": u"(GR)" + CK + N4 + N3 + C + C4 + C4 + C4 + C3,
|
||||
"GR": "(GR)" + CK + N4 + N3 + C + C4 + C4 + C4 + C3,
|
||||
# Greenland (14n) GLkk bbbb cccc cccc cc
|
||||
"GL": u"(GL)" + CK + N4 + N4 + N4 + N2,
|
||||
"GL": "(GL)" + CK + N4 + N4 + N4 + N2,
|
||||
# Guatemala (4c,20c) GTkk bbbb mmtt cccc cccc cccc cccc
|
||||
"GT": u"(GT)" + CK + C4 + C4 + C4 + C4 + C4 + C4,
|
||||
"GT": "(GT)" + CK + C4 + C4 + C4 + C4 + C4 + C4,
|
||||
# Hungary (24n) HUkk bbbs sssx cccc cccc cccc cccx
|
||||
"HU": u"(HU)" + CK + N4 + N4 + N4 + N4 + N4 + N4,
|
||||
"HU": "(HU)" + CK + N4 + N4 + N4 + N4 + N4 + N4,
|
||||
# Iceland (22n) ISkk bbbb sscc cccc iiii iiii ii
|
||||
"IS": u"(IS)" + CK + N4 + N4 + N4 + N4 + N4 + N2,
|
||||
"IS": "(IS)" + CK + N4 + N4 + N4 + N4 + N4 + N2,
|
||||
# Ireland (4c,14n) IEkk aaaa bbbb bbcc cccc cc
|
||||
"IE": u"(IE)" + CK + C4 + N4 + N4 + N4 + N2,
|
||||
"IE": "(IE)" + CK + C4 + N4 + N4 + N4 + N2,
|
||||
# Israel (19n) ILkk bbbn nncc cccc cccc ccc
|
||||
"IL": u"(IL)" + CK + N4 + N4 + N4 + N4 + N3,
|
||||
"IL": "(IL)" + CK + N4 + N4 + N4 + N4 + N3,
|
||||
# Italy (1a,10n,12c) ITkk xbbb bbss sssc cccc cccc ccc
|
||||
"IT": u"(IT)" + CK + A + N3 + N4 + N3 + C + C3 + C + C4 + C3,
|
||||
"IT": "(IT)" + CK + A + N3 + N4 + N3 + C + C3 + C + C4 + C3,
|
||||
# Jordan (4a,22n) JOkk bbbb ssss cccc cccc cccc cccc cc
|
||||
"JO": u"(JO)" + CK + A4 + N4 + N4 + N4 + N4 + N4 + N2,
|
||||
"JO": "(JO)" + CK + A4 + N4 + N4 + N4 + N4 + N4 + N2,
|
||||
# Kazakhstan (3n,13c) KZkk bbbc cccc cccc cccc
|
||||
"KZ": u"(KZ)" + CK + N3 + C + C4 + C4 + C4,
|
||||
"KZ": "(KZ)" + CK + N3 + C + C4 + C4 + C4,
|
||||
# Kosovo (4n,10n,2n) XKkk bbbb cccc cccc cccc
|
||||
"XK": u"(XK)" + CK + N4 + N4 + N4 + N4,
|
||||
"XK": "(XK)" + CK + N4 + N4 + N4 + N4,
|
||||
# Kuwait (4a,22c) KWkk bbbb cccc cccc cccc cccc cccc cc
|
||||
"KW": u"(KW)" + CK + A4 + C4 + C4 + C4 + C4 + C4 + C2,
|
||||
"KW": "(KW)" + CK + A4 + C4 + C4 + C4 + C4 + C4 + C2,
|
||||
# Latvia (4a,13c) LVkk bbbb cccc cccc cccc c
|
||||
"LV": u"(LV)" + CK + A4 + C4 + C4 + C4 + C,
|
||||
"LV": "(LV)" + CK + A4 + C4 + C4 + C4 + C,
|
||||
# Lebanon (4n,20c) LBkk bbbb cccc cccc cccc cccc cccc
|
||||
"LB": u"(LB)" + CK + N4 + C4 + C4 + C4 + C4 + C4,
|
||||
"LB": "(LB)" + CK + N4 + C4 + C4 + C4 + C4 + C4,
|
||||
# LiechteNtein (5n,12c) LIkk bbbb bccc cccc cccc c
|
||||
"LI": u"(LI)" + CK + N4 + N + C3 + C4 + C4 + C,
|
||||
"LI": "(LI)" + CK + N4 + N + C3 + C4 + C4 + C,
|
||||
# Lithuania (16n) LTkk bbbb bccc cccc cccc
|
||||
"LT": u"(LT)" + CK + N4 + N4 + N4 + N4,
|
||||
"LT": "(LT)" + CK + N4 + N4 + N4 + N4,
|
||||
# Luxembourg (3n,13c) LUkk bbbc cccc cccc cccc
|
||||
"LU": u"(LU)" + CK + N3 + C + C4 + C4 + C4,
|
||||
"LU": "(LU)" + CK + N3 + C + C4 + C4 + C4,
|
||||
# Malta (4a,5n,18c) MTkk bbbb ssss sccc cccc cccc cccc ccc
|
||||
"MT": u"(MT)" + CK + A4 + N4 + N + C3 + C4 + C4 + C4 + C3,
|
||||
"MT": "(MT)" + CK + A4 + N4 + N + C3 + C4 + C4 + C4 + C3,
|
||||
# Mauritania (23n) MRkk bbbb bsss sscc cccc cccc cxx
|
||||
"MR": u"(MR)" + CK + N4 + N4 + N4 + N4 + N4 + N3,
|
||||
"MR": "(MR)" + CK + N4 + N4 + N4 + N4 + N4 + N3,
|
||||
# Mauritius (4a,19n,3a) MUkk bbbb bbss cccc cccc cccc 000m mm
|
||||
"MU": u"(MU)" + CK + A4 + N4 + N4 + N4 + N4 + N3 + A,
|
||||
"MU": "(MU)" + CK + A4 + N4 + N4 + N4 + N4 + N3 + A,
|
||||
# Moldova (2c,18c) MDkk bbcc cccc cccc cccc cccc
|
||||
"MD": u"(MD)" + CK + C4 + C4 + C4 + C4 + C4,
|
||||
"MD": "(MD)" + CK + C4 + C4 + C4 + C4 + C4,
|
||||
# Monaco (10n,11c,2n) MCkk bbbb bsss sscc cccc cccc cxx
|
||||
"MC": u"(MC)" + CK + N4 + N4 + N2 + C2 + C4 + C4 + C + N2,
|
||||
"MC": "(MC)" + CK + N4 + N4 + N2 + C2 + C4 + C4 + C + N2,
|
||||
# Montenegro (18n) MEkk bbbc cccc cccc cccc xx
|
||||
"ME": u"(ME)" + CK + N4 + N4 + N4 + N4 + N2,
|
||||
"ME": "(ME)" + CK + N4 + N4 + N4 + N4 + N2,
|
||||
# Netherlands (4a,10n) NLkk bbbb cccc cccc cc
|
||||
"NL": u"(NL)" + CK + A4 + N4 + N4 + N2,
|
||||
"NL": "(NL)" + CK + A4 + N4 + N4 + N2,
|
||||
# North Macedonia (3n,10c,2n) MKkk bbbc cccc cccc cxx
|
||||
"MK": u"(MK)" + CK + N3 + C + C4 + C4 + C + N2,
|
||||
"MK": "(MK)" + CK + N3 + C + C4 + C4 + C + N2,
|
||||
# Norway (11n) NOkk bbbb cccc ccx
|
||||
"NO": u"(NO)" + CK + N4 + N4 + N3,
|
||||
"NO": "(NO)" + CK + N4 + N4 + N3,
|
||||
# Pakistan (4c,16n) PKkk bbbb cccc cccc cccc cccc
|
||||
"PK": u"(PK)" + CK + C4 + N4 + N4 + N4 + N4,
|
||||
"PK": "(PK)" + CK + C4 + N4 + N4 + N4 + N4,
|
||||
# Palestinian territories (4c,21n) PSkk bbbb xxxx xxxx xccc cccc cccc c
|
||||
"PS": u"(PS)" + CK + C4 + N4 + N4 + N4 + N4 + N,
|
||||
"PS": "(PS)" + CK + C4 + N4 + N4 + N4 + N4 + N,
|
||||
# Poland (24n) PLkk bbbs sssx cccc cccc cccc cccc
|
||||
"PL": u"(PL)" + CK + N4 + N4 + N4 + N4 + N4 + N4,
|
||||
"PL": "(PL)" + CK + N4 + N4 + N4 + N4 + N4 + N4,
|
||||
# Portugal (21n) PTkk bbbb ssss cccc cccc cccx x
|
||||
"PT": u"(PT)" + CK + N4 + N4 + N4 + N4 + N,
|
||||
"PT": "(PT)" + CK + N4 + N4 + N4 + N4 + N,
|
||||
# Qatar (4a,21c) QAkk bbbb cccc cccc cccc cccc cccc c
|
||||
"QA": u"(QA)" + CK + A4 + C4 + C4 + C4 + C4 + C,
|
||||
"QA": "(QA)" + CK + A4 + C4 + C4 + C4 + C4 + C,
|
||||
# Romania (4a,16c) ROkk bbbb cccc cccc cccc cccc
|
||||
"RO": u"(RO)" + CK + A4 + C4 + C4 + C4 + C4,
|
||||
"RO": "(RO)" + CK + A4 + C4 + C4 + C4 + C4,
|
||||
# San Marino (1a,10n,12c) SMkk xbbb bbss sssc cccc cccc ccc
|
||||
"SM": u"(SM)" + CK + A + N3 + N4 + N3 + C + C4 + C4 + C3,
|
||||
"SM": "(SM)" + CK + A + N3 + N4 + N3 + C + C4 + C4 + C3,
|
||||
# Saudi Arabia (2n,18c) SAkk bbcc cccc cccc cccc cccc
|
||||
"SA": u"(SA)" + CK + N2 + C2 + C4 + C4 + C4 + C4,
|
||||
"SA": "(SA)" + CK + N2 + C2 + C4 + C4 + C4 + C4,
|
||||
# Serbia (18n) RSkk bbbc cccc cccc cccc xx
|
||||
"RS": u"(RS)" + CK + N4 + N4 + N4 + N4 + N2,
|
||||
"RS": "(RS)" + CK + N4 + N4 + N4 + N4 + N2,
|
||||
# Slovakia (20n) SKkk bbbb ssss sscc cccc cccc
|
||||
"SK": u"(SK)" + CK + N4 + N4 + N4 + N4 + N4,
|
||||
"SK": "(SK)" + CK + N4 + N4 + N4 + N4 + N4,
|
||||
# Slovenia (15n) SIkk bbss sccc cccc cxx
|
||||
"SI": u"(SI)" + CK + N4 + N4 + N4 + N3,
|
||||
"SI": "(SI)" + CK + N4 + N4 + N4 + N3,
|
||||
# Spain (20n) ESkk bbbb ssss xxcc cccc cccc
|
||||
"ES": u"(ES)" + CK + N4 + N4 + N4 + N4 + N4,
|
||||
"ES": "(ES)" + CK + N4 + N4 + N4 + N4 + N4,
|
||||
# Sweden (20n) SEkk bbbc cccc cccc cccc cccc
|
||||
"SE": u"(SE)" + CK + N4 + N4 + N4 + N4 + N4,
|
||||
"SE": "(SE)" + CK + N4 + N4 + N4 + N4 + N4,
|
||||
# Switzerland (5n,12c) CHkk bbbb bccc cccc cccc c
|
||||
"CH": u"(CH)" + CK + N4 + N + C3 + C4 + C4 + C,
|
||||
"CH": "(CH)" + CK + N4 + N + C3 + C4 + C4 + C,
|
||||
# Tunisia (20n) TNkk bbss sccc cccc cccc cccc
|
||||
"TN": u"(TN)" + CK + N4 + N4 + N4 + N4 + N4,
|
||||
"TN": "(TN)" + CK + N4 + N4 + N4 + N4 + N4,
|
||||
# Turkey (5n,17c) TRkk bbbb bxcc cccc cccc cccc cc
|
||||
"TR": u"(TR)" + CK + N4 + N + C3 + C4 + C4 + C4 + C2,
|
||||
"TR": "(TR)" + CK + N4 + N + C3 + C4 + C4 + C4 + C2,
|
||||
# United Arab Emirates (3n,16n) AEkk bbbc cccc cccc cccc ccc
|
||||
"AE": u"(AE)" + CK + N4 + N4 + N4 + N4 + N3,
|
||||
"AE": "(AE)" + CK + N4 + N4 + N4 + N4 + N3,
|
||||
# United Kingdom (4a,14n) GBkk bbbb ssss sscc cccc cc
|
||||
"GB": u"(GB)" + CK + A4 + N4 + N4 + N4 + N2,
|
||||
"GB": "(GB)" + CK + A4 + N4 + N4 + N4 + N2,
|
||||
# Vatican City (3n,15n) VAkk bbbc cccc cccc cccc cc
|
||||
"VA": u"(VA)" + CK + N4 + N4 + N4 + N4 + N2,
|
||||
"VA": "(VA)" + CK + N4 + N4 + N4 + N4 + N2,
|
||||
# Virgin Islands, British (4c,16n) VGkk bbbb cccc cccc cccc cccc
|
||||
"VG": u"(VG)" + CK + C4 + N4 + N4 + N4 + N4,
|
||||
"VG": "(VG)" + CK + C4 + N4 + N4 + N4 + N4,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import re
|
||||
from typing import List
|
||||
|
||||
import pytest
|
||||
|
|
@ -5,8 +6,6 @@ import pytest
|
|||
from presidio_analyzer import Pattern, RecognizerResult
|
||||
from presidio_analyzer import PatternRecognizer
|
||||
|
||||
# https://www.datatrans.ch/showcase/test-cc-numbers
|
||||
# https://www.freeformatter.com/credit-card-number-generator-validator.html
|
||||
from tests import assert_result
|
||||
|
||||
|
||||
|
|
@ -127,3 +126,78 @@ def test_when_validation_occurs_then_analysis_explanation_is_updated():
|
|||
|
||||
assert results[0].analysis_explanation.original_score == 0.5
|
||||
assert results[0].analysis_explanation.score == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text, expected_len, deny_list",
|
||||
[
|
||||
("Mr. PLUM", 1, ["Mr.", "Mrs."]),
|
||||
("...Mr...PLUM...", 1, ["Mr.", "Mrs."]),
|
||||
("..MMr...PLUM...", 0, ["Mr.", "Mrs."]),
|
||||
("Mrr PLUM...", 0, ["Mr.", "Mrs."]),
|
||||
("\\Mr.\\ PLUM...", 1, ["Mr.", "Mrs."]),
|
||||
("\\Mr.\\ PLUM...,Mrs. Plum", 2, ["Mr.", "Mrs."]),
|
||||
("", 0, ["Mr.", "Mrs."]),
|
||||
("MMrrrMrs.", 0, ["Mr.", "Mrs."]),
|
||||
("\\Mrs.", 1, ["Mr.", "Mrs."]),
|
||||
("A B is an entity", 1, ["A B", "B C"]),
|
||||
("A is not an entity", 0, ["A B", "B C"]),
|
||||
("A B C", 1, ["A B", "B C"]),
|
||||
("A B B C", 2, ["A B", "B C"]),
|
||||
("Hi A.,.\\.B Hi", 1, ["A.,.\\.B"]),
|
||||
],
|
||||
)
|
||||
def test_deny_list_non_space_separator_identified_correctly(
|
||||
text, expected_len, deny_list
|
||||
):
|
||||
recognizer = PatternRecognizer(
|
||||
supported_entity="TITLE",
|
||||
name="TitlesRecognizer",
|
||||
supported_language="en",
|
||||
deny_list=deny_list,
|
||||
)
|
||||
|
||||
result = recognizer.analyze(text, entities=["TITLE"])
|
||||
|
||||
assert len(result) == expected_len
|
||||
|
||||
|
||||
def test_deny_list_score_change():
|
||||
deny_list = ["Mr.", "Mrs."]
|
||||
recognizer = PatternRecognizer(
|
||||
supported_entity="TITLE",
|
||||
name="TitlesRecognizer",
|
||||
supported_language="en",
|
||||
deny_list=deny_list,
|
||||
deny_list_score=0.64,
|
||||
)
|
||||
|
||||
result = recognizer.analyze(text="Mrs. Kennedy", entities=["TITLE"])
|
||||
assert result[0].score == 0.64
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text,flag,expected_len",
|
||||
[("mrs. Kennedy", re.IGNORECASE, 1), ("mrs. Kennedy", None, 0)],
|
||||
)
|
||||
def test_deny_list_regex_flags(text, flag, expected_len):
|
||||
deny_list = ["Mr.", "Mrs."]
|
||||
recognizer = PatternRecognizer(
|
||||
supported_entity="TITLE",
|
||||
name="TitlesRecognizer",
|
||||
supported_language="en",
|
||||
deny_list=deny_list,
|
||||
)
|
||||
|
||||
result = recognizer.analyze(text=text, entities=["TITLE"], regex_flags=flag)
|
||||
assert len(result) == expected_len
|
||||
|
||||
|
||||
def test_empty_deny_list_raises_value_error():
|
||||
with pytest.raises(ValueError):
|
||||
PatternRecognizer(
|
||||
supported_entity="TITLE",
|
||||
name="TitlesRecognizer",
|
||||
supported_language="en",
|
||||
deny_list=[],
|
||||
)
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче