From 8c661fef344f57bc01a88575353d7c3cab7c74e4 Mon Sep 17 00:00:00 2001 From: Omri Mendels Date: Mon, 2 May 2022 17:05:38 +0300 Subject: [PATCH] Updates to deny_list recognizer (#858) --- e2e-tests/tests/test_analyzer.py | 4 +- .../nlp_engine/nlp_artifacts.py | 2 +- .../presidio_analyzer/pattern_recognizer.py | 20 +- .../predefined_recognizers/iban_patterns.py | 172 +++++++++--------- .../tests/test_pattern_recognizer.py | 78 +++++++- 5 files changed, 178 insertions(+), 98 deletions(-) diff --git a/e2e-tests/tests/test_analyzer.py b/e2e-tests/tests/test_analyzer.py index f9ecb253..600ab9c1 100644 --- a/e2e-tests/tests/test_analyzer.py +++ b/e2e-tests/tests/test_analyzer.py @@ -445,13 +445,13 @@ def test_given_ad_hoc_deny_list_recognizer_the_right_entities_are_returned(): { "name": "Mr. Recognizer", "supported_language": "en", - "deny_list": ["Mr", "Mr.", "Mister"], + "deny_list": ["Mr.", "Mister"], "supported_entity":"MR_TITLE" }, { "name": "Ms. Recognizer", "supported_language": "en", - "deny_list": ["Ms", "Ms.", "Miss", "Mrs", "Mrs."], + "deny_list": ["Ms.", "Miss", "Mrs."], "supported_entity":"MS_TITLE" } ] diff --git a/presidio-analyzer/presidio_analyzer/nlp_engine/nlp_artifacts.py b/presidio-analyzer/presidio_analyzer/nlp_engine/nlp_artifacts.py index 61b350e9..d2862477 100644 --- a/presidio-analyzer/presidio_analyzer/nlp_engine/nlp_artifacts.py +++ b/presidio-analyzer/presidio_analyzer/nlp_engine/nlp_artifacts.py @@ -63,7 +63,7 @@ class NlpArtifacts: return_dict = self.__dict__.copy() # Ignore NLP engine as it's not serializable currently - del return_dict['nlp_engine'] + del return_dict["nlp_engine"] # Converting spaCy tokens and spans to string as they are not serializable if "tokens" in return_dict: diff --git a/presidio-analyzer/presidio_analyzer/pattern_recognizer.py b/presidio-analyzer/presidio_analyzer/pattern_recognizer.py index 22801ce0..379c6573 100644 --- a/presidio-analyzer/presidio_analyzer/pattern_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/pattern_recognizer.py @@ -24,6 +24,8 @@ class PatternRecognizer(LocalRecognizer): :param deny_list: A list of words to detect, in case our recognizer uses a predefined list of words (deny list) :param context: list of context words + :param deny_list_score: confidence score for a term + identified using a deny-list """ def __init__( @@ -34,6 +36,7 @@ class PatternRecognizer(LocalRecognizer): patterns: List[Pattern] = None, deny_list: List[str] = None, context: List[str] = None, + deny_list_score: float = 1.0, version: str = "0.0.1", ): @@ -57,9 +60,10 @@ class PatternRecognizer(LocalRecognizer): else: self.patterns = patterns self.context = context + self.deny_list_score = deny_list_score if deny_list: - deny_list_pattern = self.__deny_list_to_regex(deny_list) + deny_list_pattern = self._deny_list_to_regex(deny_list) self.patterns.append(deny_list_pattern) self.deny_list = deny_list else: @@ -92,18 +96,20 @@ class PatternRecognizer(LocalRecognizer): return results - @staticmethod - def __deny_list_to_regex(deny_list: List[str]) -> Pattern: + def _deny_list_to_regex(self, deny_list: List[str]) -> Pattern: """ - Convert a list of word to a matching regex. + Convert a list of words to a matching regex. - To be analyzed by the regex engine as a part of the analyze logic. + To be analyzed by the analyze method as any other regex patterns. :param deny_list: the list of words to detect :return:the regex of the words for detection """ - regex = r"(?:^|(?<= ))(" + "|".join(deny_list) + r")(?:(?= )|$)" - return Pattern(name="deny_list", regex=regex, score=1.0) + + # Escape deny list elements as preparation for regex + escaped_deny_list = [re.escape(element) for element in deny_list] + regex = r"(?:^|(?<=\W))(" + "|".join(escaped_deny_list) + r")(?:(?=\W)|$)" + return Pattern(name="deny_list", regex=regex, score=self.deny_list_score) def validate_result(self, pattern_text: str) -> Optional[bool]: """ diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/iban_patterns.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/iban_patterns.py index c892e2dc..a2d7ecd9 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/iban_patterns.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/iban_patterns.py @@ -9,165 +9,165 @@ An IBAN checker is available here: https://www.iban.com/iban-checker """ # IBAN parts format -CC = u"[A-Z]{2}" # country code -CK = u"[0-9]{2}[ ]?" # checksum -BOS = u"^" -EOS = u"$" # end of string +CC = "[A-Z]{2}" # country code +CK = "[0-9]{2}[ ]?" # checksum +BOS = "^" +EOS = "$" # end of string -A = u"[A-Z][ ]?" -A2 = u"([A-Z][ ]?){2}" -A3 = u"([A-Z][ ]?){3}" -A4 = u"([A-Z][ ]?){4}" +A = "[A-Z][ ]?" +A2 = "([A-Z][ ]?){2}" +A3 = "([A-Z][ ]?){3}" +A4 = "([A-Z][ ]?){4}" -C = u"[a-zA-Z0-9][ ]?" -C2 = u"([a-zA-Z0-9][ ]?){2}" -C3 = u"([a-zA-Z0-9][ ]?){3}" -C4 = u"([a-zA-Z0-9][ ]?){4}" +C = "[a-zA-Z0-9][ ]?" +C2 = "([a-zA-Z0-9][ ]?){2}" +C3 = "([a-zA-Z0-9][ ]?){3}" +C4 = "([a-zA-Z0-9][ ]?){4}" -N = u"[0-9][ ]?" -N2 = u"([0-9][ ]?){2}" -N3 = u"([0-9][ ]?){3}" -N4 = u"([0-9][ ]?){4}" +N = "[0-9][ ]?" +N2 = "([0-9][ ]?){2}" +N3 = "([0-9][ ]?){3}" +N4 = "([0-9][ ]?){4}" regex_per_country = { # Albania (8n, 16c) ALkk bbbs sssx cccc cccc cccc cccc - "AL": u"(AL)" + CK + N4 + N4 + C4 + C4 + C4 + C4, + "AL": "(AL)" + CK + N4 + N4 + C4 + C4 + C4 + C4, # Andorra (8n, 12c) ADkk bbbb ssss cccc cccc cccc - "AD": u"(AD)" + CK + N4 + N4 + C4 + C4 + C4, + "AD": "(AD)" + CK + N4 + N4 + C4 + C4 + C4, # Austria (16n) ATkk bbbb bccc cccc cccc - "AT": u"(AT)" + CK + N4 + N4 + N4 + N4, + "AT": "(AT)" + CK + N4 + N4 + N4 + N4, # Azerbaijan    (4c,20n) AZkk bbbb cccc cccc cccc cccc cccc - "AZ": u"(AZ)" + CK + C4 + N4 + N4 + N4 + N4 + N4, + "AZ": "(AZ)" + CK + C4 + N4 + N4 + N4 + N4 + N4, # Bahrain   (4a,14c)    BHkk bbbb cccc cccc cccc cc - "BH": u"(BH)" + CK + A4 + C4 + C4 + C4 + C2, + "BH": "(BH)" + CK + A4 + C4 + C4 + C4 + C2, # Belarus (4c, 4n, 16c)   BYkk bbbb aaaa cccc cccc cccc cccc - "BY": u"(BY)" + CK + C4 + N4 + C4 + C4 + C4 + C4, + "BY": "(BY)" + CK + C4 + N4 + C4 + C4 + C4 + C4, # Belgium (12n)   BEkk bbbc cccc ccxx - "BE": u"(BE)" + CK + N4 + N4 + N4, + "BE": "(BE)" + CK + N4 + N4 + N4, # Bosnia and Herzegovina    (16n)   BAkk bbbs sscc cccc ccxx - "BA": u"(BA)" + CK + N4 + N4 + N4 + N4, + "BA": "(BA)" + CK + N4 + N4 + N4 + N4, # Brazil (23n,1a,1c) BRkk bbbb bbbb ssss sccc cccc ccct n - "BR": u"(BR)" + CK + N4 + N4 + N4 + N4 + N4 + N3 + A + C, + "BR": "(BR)" + CK + N4 + N4 + N4 + N4 + N4 + N3 + A + C, # Bulgaria  (4a,6n,8c)  BGkk bbbb ssss ttcc cccc cc - "BG": u"(BG)" + CK + A4 + N4 + N + N + C2 + C4 + C2, + "BG": "(BG)" + CK + A4 + N4 + N + N + C2 + C4 + C2, # Costa Rica (18n) CRkk 0bbb cccc cccc cccc cc (0 = always zero) - "CR": u"(CR)" + CK + u"[0]" + N3 + N4 + N4 + N4 + N2, + "CR": "(CR)" + CK + "[0]" + N3 + N4 + N4 + N4 + N2, # Croatia (17n) HRkk bbbb bbbc cccc cccc c - "HR": u"(HR)" + CK + N4 + N4 + N4 + N4 + N, + "HR": "(HR)" + CK + N4 + N4 + N4 + N4 + N, # Cyprus (8n,16c) CYkk bbbs ssss cccc cccc cccc cccc - "CY": u"(CY)" + CK + N4 + N4 + C4 + C4 + C4 + C4, + "CY": "(CY)" + CK + N4 + N4 + C4 + C4 + C4 + C4, # Czech Republic (20n) CZkk bbbb ssss sscc cccc cccc - "CZ": u"(CZ)" + CK + N4 + N4 + N4 + N4 + N4, + "CZ": "(CZ)" + CK + N4 + N4 + N4 + N4 + N4, # Denmark (14n) DKkk bbbb cccc cccc cc - "DK": u"(DK)" + CK + N4 + N4 + N4 + N2, + "DK": "(DK)" + CK + N4 + N4 + N4 + N2, # Dominican Republic (4a,20n) DOkk bbbb cccc cccc cccc cccc cccc - "DO": u"(DO)" + CK + A4 + N4 + N4 + N4 + N4 + N4, + "DO": "(DO)" + CK + A4 + N4 + N4 + N4 + N4 + N4, # EAt Timor (19n) TLkk bbbc cccc cccc cccc cxx - "TL": u"(TL)" + CK + N4 + N4 + N4 + N4 + N3, + "TL": "(TL)" + CK + N4 + N4 + N4 + N4 + N3, # Estonia (16n) EEkk bbss cccc cccc cccx - "EE": u"(EE)" + CK + N4 + N4 + N4 + N4, + "EE": "(EE)" + CK + N4 + N4 + N4 + N4, # Faroe Islands (14n) FOkk bbbb cccc cccc cx - "FO": u"(FO)" + CK + N4 + N4 + N4 + N2, + "FO": "(FO)" + CK + N4 + N4 + N4 + N2, # Finland (14n) FIkk bbbb bbcc cccc cx - "FI": u"(FI)" + CK + N4 + N4 + N4 + N2, + "FI": "(FI)" + CK + N4 + N4 + N4 + N2, # France (10n,11c,2n) FRkk bbbb bsss sscc cccc cccc cxx - "FR": u"(FR)" + CK + N4 + N4 + N2 + C2 + C4 + C4 + C + N2, + "FR": "(FR)" + CK + N4 + N4 + N2 + C2 + C4 + C4 + C + N2, # Georgia (2c,16n)  GEkk bbcc cccc cccc cccc cc - "GE": u"(GE)" + CK + C2 + N2 + N4 + N4 + N4 + N2, + "GE": "(GE)" + CK + C2 + N2 + N4 + N4 + N4 + N2, # Germany (18n) DEkk bbbb bbbb cccc cccc cc - "DE": u"(DE)" + CK + N4 + N4 + N4 + N4 + N2, + "DE": "(DE)" + CK + N4 + N4 + N4 + N4 + N2, # Gibraltar (4a,15c)  GIkk bbbb cccc cccc cccc ccc - "GI": u"(GI)" + CK + A4 + C4 + C4 + C4 + C3, + "GI": "(GI)" + CK + A4 + C4 + C4 + C4 + C3, # Greece (7n,16c)  GRkk bbbs sssc cccc cccc cccc ccc - "GR": u"(GR)" + CK + N4 + N3 + C + C4 + C4 + C4 + C3, + "GR": "(GR)" + CK + N4 + N3 + C + C4 + C4 + C4 + C3, # Greenland (14n) GLkk bbbb cccc cccc cc - "GL": u"(GL)" + CK + N4 + N4 + N4 + N2, + "GL": "(GL)" + CK + N4 + N4 + N4 + N2, # Guatemala (4c,20c)  GTkk bbbb mmtt cccc cccc cccc cccc - "GT": u"(GT)" + CK + C4 + C4 + C4 + C4 + C4 + C4, + "GT": "(GT)" + CK + C4 + C4 + C4 + C4 + C4 + C4, # Hungary (24n) HUkk bbbs sssx cccc cccc cccc cccx - "HU": u"(HU)" + CK + N4 + N4 + N4 + N4 + N4 + N4, + "HU": "(HU)" + CK + N4 + N4 + N4 + N4 + N4 + N4, # Iceland (22n) ISkk bbbb sscc cccc iiii iiii ii - "IS": u"(IS)" + CK + N4 + N4 + N4 + N4 + N4 + N2, + "IS": "(IS)" + CK + N4 + N4 + N4 + N4 + N4 + N2, # Ireland (4c,14n)  IEkk aaaa bbbb bbcc cccc cc - "IE": u"(IE)" + CK + C4 + N4 + N4 + N4 + N2, + "IE": "(IE)" + CK + C4 + N4 + N4 + N4 + N2, # Israel (19n) ILkk bbbn nncc cccc cccc ccc - "IL": u"(IL)" + CK + N4 + N4 + N4 + N4 + N3, + "IL": "(IL)" + CK + N4 + N4 + N4 + N4 + N3, # Italy (1a,10n,12c)  ITkk xbbb bbss sssc cccc cccc ccc - "IT": u"(IT)" + CK + A + N3 + N4 + N3 + C + C3 + C + C4 + C3, + "IT": "(IT)" + CK + A + N3 + N4 + N3 + C + C3 + C + C4 + C3, # Jordan (4a,22n)  JOkk bbbb ssss cccc cccc cccc cccc cc - "JO": u"(JO)" + CK + A4 + N4 + N4 + N4 + N4 + N4 + N2, + "JO": "(JO)" + CK + A4 + N4 + N4 + N4 + N4 + N4 + N2, # Kazakhstan (3n,13c)  KZkk bbbc cccc cccc cccc - "KZ": u"(KZ)" + CK + N3 + C + C4 + C4 + C4, + "KZ": "(KZ)" + CK + N3 + C + C4 + C4 + C4, # Kosovo (4n,10n,2n)   XKkk bbbb cccc cccc cccc - "XK": u"(XK)" + CK + N4 + N4 + N4 + N4, + "XK": "(XK)" + CK + N4 + N4 + N4 + N4, # Kuwait (4a,22c)  KWkk bbbb cccc cccc cccc cccc cccc cc - "KW": u"(KW)" + CK + A4 + C4 + C4 + C4 + C4 + C4 + C2, + "KW": "(KW)" + CK + A4 + C4 + C4 + C4 + C4 + C4 + C2, # Latvia (4a,13c)  LVkk bbbb cccc cccc cccc c - "LV": u"(LV)" + CK + A4 + C4 + C4 + C4 + C, + "LV": "(LV)" + CK + A4 + C4 + C4 + C4 + C, # Lebanon (4n,20c)  LBkk bbbb cccc cccc cccc cccc cccc - "LB": u"(LB)" + CK + N4 + C4 + C4 + C4 + C4 + C4, + "LB": "(LB)" + CK + N4 + C4 + C4 + C4 + C4 + C4, # LiechteNtein (5n,12c)  LIkk bbbb bccc cccc cccc c - "LI": u"(LI)" + CK + N4 + N + C3 + C4 + C4 + C, + "LI": "(LI)" + CK + N4 + N + C3 + C4 + C4 + C, # Lithuania (16n) LTkk bbbb bccc cccc cccc - "LT": u"(LT)" + CK + N4 + N4 + N4 + N4, + "LT": "(LT)" + CK + N4 + N4 + N4 + N4, # Luxembourg (3n,13c)  LUkk bbbc cccc cccc cccc - "LU": u"(LU)" + CK + N3 + C + C4 + C4 + C4, + "LU": "(LU)" + CK + N3 + C + C4 + C4 + C4, # Malta (4a,5n,18c)   MTkk bbbb ssss sccc cccc cccc cccc ccc - "MT": u"(MT)" + CK + A4 + N4 + N + C3 + C4 + C4 + C4 + C3, + "MT": "(MT)" + CK + A4 + N4 + N + C3 + C4 + C4 + C4 + C3, # Mauritania (23n) MRkk bbbb bsss sscc cccc cccc cxx - "MR": u"(MR)" + CK + N4 + N4 + N4 + N4 + N4 + N3, + "MR": "(MR)" + CK + N4 + N4 + N4 + N4 + N4 + N3, # Mauritius (4a,19n,3a)   MUkk bbbb bbss cccc cccc cccc 000m mm - "MU": u"(MU)" + CK + A4 + N4 + N4 + N4 + N4 + N3 + A, + "MU": "(MU)" + CK + A4 + N4 + N4 + N4 + N4 + N3 + A, # Moldova (2c,18c)  MDkk bbcc cccc cccc cccc cccc - "MD": u"(MD)" + CK + C4 + C4 + C4 + C4 + C4, + "MD": "(MD)" + CK + C4 + C4 + C4 + C4 + C4, # Monaco (10n,11c,2n)  MCkk bbbb bsss sscc cccc cccc cxx - "MC": u"(MC)" + CK + N4 + N4 + N2 + C2 + C4 + C4 + C + N2, + "MC": "(MC)" + CK + N4 + N4 + N2 + C2 + C4 + C4 + C + N2, # Montenegro (18n) MEkk bbbc cccc cccc cccc xx - "ME": u"(ME)" + CK + N4 + N4 + N4 + N4 + N2, + "ME": "(ME)" + CK + N4 + N4 + N4 + N4 + N2, # Netherlands (4a,10n)  NLkk bbbb cccc cccc cc - "NL": u"(NL)" + CK + A4 + N4 + N4 + N2, + "NL": "(NL)" + CK + A4 + N4 + N4 + N2, # North Macedonia (3n,10c,2n)   MKkk bbbc cccc cccc cxx - "MK": u"(MK)" + CK + N3 + C + C4 + C4 + C + N2, + "MK": "(MK)" + CK + N3 + C + C4 + C4 + C + N2, # Norway (11n) NOkk bbbb cccc ccx - "NO": u"(NO)" + CK + N4 + N4 + N3, + "NO": "(NO)" + CK + N4 + N4 + N3, # Pakistan  (4c,16n)  PKkk bbbb cccc cccc cccc cccc - "PK": u"(PK)" + CK + C4 + N4 + N4 + N4 + N4, + "PK": "(PK)" + CK + C4 + N4 + N4 + N4 + N4, # Palestinian territories (4c,21n)  PSkk bbbb xxxx xxxx xccc cccc cccc c - "PS": u"(PS)" + CK + C4 + N4 + N4 + N4 + N4 + N, + "PS": "(PS)" + CK + C4 + N4 + N4 + N4 + N4 + N, # Poland (24n) PLkk bbbs sssx cccc cccc cccc cccc - "PL": u"(PL)" + CK + N4 + N4 + N4 + N4 + N4 + N4, + "PL": "(PL)" + CK + N4 + N4 + N4 + N4 + N4 + N4, # Portugal (21n) PTkk bbbb ssss cccc cccc cccx x - "PT": u"(PT)" + CK + N4 + N4 + N4 + N4 + N, + "PT": "(PT)" + CK + N4 + N4 + N4 + N4 + N, # Qatar (4a,21c)  QAkk bbbb cccc cccc cccc cccc cccc c - "QA": u"(QA)" + CK + A4 + C4 + C4 + C4 + C4 + C, + "QA": "(QA)" + CK + A4 + C4 + C4 + C4 + C4 + C, # Romania (4a,16c)  ROkk bbbb cccc cccc cccc cccc - "RO": u"(RO)" + CK + A4 + C4 + C4 + C4 + C4, + "RO": "(RO)" + CK + A4 + C4 + C4 + C4 + C4, # San Marino (1a,10n,12c)  SMkk xbbb bbss sssc cccc cccc ccc - "SM": u"(SM)" + CK + A + N3 + N4 + N3 + C + C4 + C4 + C3, + "SM": "(SM)" + CK + A + N3 + N4 + N3 + C + C4 + C4 + C3, # Saudi Arabia (2n,18c)  SAkk bbcc cccc cccc cccc cccc - "SA": u"(SA)" + CK + N2 + C2 + C4 + C4 + C4 + C4, + "SA": "(SA)" + CK + N2 + C2 + C4 + C4 + C4 + C4, # Serbia (18n) RSkk bbbc cccc cccc cccc xx - "RS": u"(RS)" + CK + N4 + N4 + N4 + N4 + N2, + "RS": "(RS)" + CK + N4 + N4 + N4 + N4 + N2, # Slovakia (20n) SKkk bbbb ssss sscc cccc cccc - "SK": u"(SK)" + CK + N4 + N4 + N4 + N4 + N4, + "SK": "(SK)" + CK + N4 + N4 + N4 + N4 + N4, # Slovenia (15n) SIkk bbss sccc cccc cxx - "SI": u"(SI)" + CK + N4 + N4 + N4 + N3, + "SI": "(SI)" + CK + N4 + N4 + N4 + N3, # Spain (20n) ESkk bbbb ssss xxcc cccc cccc - "ES": u"(ES)" + CK + N4 + N4 + N4 + N4 + N4, + "ES": "(ES)" + CK + N4 + N4 + N4 + N4 + N4, # Sweden (20n) SEkk bbbc cccc cccc cccc cccc - "SE": u"(SE)" + CK + N4 + N4 + N4 + N4 + N4, + "SE": "(SE)" + CK + N4 + N4 + N4 + N4 + N4, # Switzerland (5n,12c)  CHkk bbbb bccc cccc cccc c - "CH": u"(CH)" + CK + N4 + N + C3 + C4 + C4 + C, + "CH": "(CH)" + CK + N4 + N + C3 + C4 + C4 + C, # Tunisia (20n) TNkk bbss sccc cccc cccc cccc - "TN": u"(TN)" + CK + N4 + N4 + N4 + N4 + N4, + "TN": "(TN)" + CK + N4 + N4 + N4 + N4 + N4, # Turkey (5n,17c)  TRkk bbbb bxcc cccc cccc cccc cc - "TR": u"(TR)" + CK + N4 + N + C3 + C4 + C4 + C4 + C2, + "TR": "(TR)" + CK + N4 + N + C3 + C4 + C4 + C4 + C2, # United Arab Emirates (3n,16n)  AEkk bbbc cccc cccc cccc ccc - "AE": u"(AE)" + CK + N4 + N4 + N4 + N4 + N3, + "AE": "(AE)" + CK + N4 + N4 + N4 + N4 + N3, # United Kingdom (4a,14n) GBkk bbbb ssss sscc cccc cc - "GB": u"(GB)" + CK + A4 + N4 + N4 + N4 + N2, + "GB": "(GB)" + CK + A4 + N4 + N4 + N4 + N2, # Vatican City (3n,15n)  VAkk bbbc cccc cccc cccc cc - "VA": u"(VA)" + CK + N4 + N4 + N4 + N4 + N2, + "VA": "(VA)" + CK + N4 + N4 + N4 + N4 + N2, # Virgin Islands, British (4c,16n)  VGkk bbbb cccc cccc cccc cccc - "VG": u"(VG)" + CK + C4 + N4 + N4 + N4 + N4, + "VG": "(VG)" + CK + C4 + N4 + N4 + N4 + N4, } diff --git a/presidio-analyzer/tests/test_pattern_recognizer.py b/presidio-analyzer/tests/test_pattern_recognizer.py index 6300cb62..426d5e6f 100644 --- a/presidio-analyzer/tests/test_pattern_recognizer.py +++ b/presidio-analyzer/tests/test_pattern_recognizer.py @@ -1,3 +1,4 @@ +import re from typing import List import pytest @@ -5,8 +6,6 @@ import pytest from presidio_analyzer import Pattern, RecognizerResult from presidio_analyzer import PatternRecognizer -# https://www.datatrans.ch/showcase/test-cc-numbers -# https://www.freeformatter.com/credit-card-number-generator-validator.html from tests import assert_result @@ -127,3 +126,78 @@ def test_when_validation_occurs_then_analysis_explanation_is_updated(): assert results[0].analysis_explanation.original_score == 0.5 assert results[0].analysis_explanation.score == 1 + + +@pytest.mark.parametrize( + "text, expected_len, deny_list", + [ + ("Mr. PLUM", 1, ["Mr.", "Mrs."]), + ("...Mr...PLUM...", 1, ["Mr.", "Mrs."]), + ("..MMr...PLUM...", 0, ["Mr.", "Mrs."]), + ("Mrr PLUM...", 0, ["Mr.", "Mrs."]), + ("\\Mr.\\ PLUM...", 1, ["Mr.", "Mrs."]), + ("\\Mr.\\ PLUM...,Mrs. Plum", 2, ["Mr.", "Mrs."]), + ("", 0, ["Mr.", "Mrs."]), + ("MMrrrMrs.", 0, ["Mr.", "Mrs."]), + ("\\Mrs.", 1, ["Mr.", "Mrs."]), + ("A B is an entity", 1, ["A B", "B C"]), + ("A is not an entity", 0, ["A B", "B C"]), + ("A B C", 1, ["A B", "B C"]), + ("A B B C", 2, ["A B", "B C"]), + ("Hi A.,.\\.B Hi", 1, ["A.,.\\.B"]), + ], +) +def test_deny_list_non_space_separator_identified_correctly( + text, expected_len, deny_list +): + recognizer = PatternRecognizer( + supported_entity="TITLE", + name="TitlesRecognizer", + supported_language="en", + deny_list=deny_list, + ) + + result = recognizer.analyze(text, entities=["TITLE"]) + + assert len(result) == expected_len + + +def test_deny_list_score_change(): + deny_list = ["Mr.", "Mrs."] + recognizer = PatternRecognizer( + supported_entity="TITLE", + name="TitlesRecognizer", + supported_language="en", + deny_list=deny_list, + deny_list_score=0.64, + ) + + result = recognizer.analyze(text="Mrs. Kennedy", entities=["TITLE"]) + assert result[0].score == 0.64 + + +@pytest.mark.parametrize( + "text,flag,expected_len", + [("mrs. Kennedy", re.IGNORECASE, 1), ("mrs. Kennedy", None, 0)], +) +def test_deny_list_regex_flags(text, flag, expected_len): + deny_list = ["Mr.", "Mrs."] + recognizer = PatternRecognizer( + supported_entity="TITLE", + name="TitlesRecognizer", + supported_language="en", + deny_list=deny_list, + ) + + result = recognizer.analyze(text=text, entities=["TITLE"], regex_flags=flag) + assert len(result) == expected_len + + +def test_empty_deny_list_raises_value_error(): + with pytest.raises(ValueError): + PatternRecognizer( + supported_entity="TITLE", + name="TitlesRecognizer", + supported_language="en", + deny_list=[], + )