From e2acf81734a6b568eff0c590ced7c8624b111713 Mon Sep 17 00:00:00 2001 From: Shiven Chawla Date: Wed, 7 Dec 2022 11:45:51 -0800 Subject: [PATCH] create test framework (#67) Co-authored-by: Shiven Chawla --- .coverage | Bin 0 -> 53248 bytes .gitignore | 1 - conftest.py | 28 +++++++++++ test_config.yml | 5 -- tests/attacks/art/test_blackbox_rule_based.py | 13 +---- tests/attacks/art/test_hop_skip_jump.py | 10 ++-- .../art/test_label_only_boundary_distance.py | 8 +-- tests/attacks/art/test_mi_face.py | 2 +- tests/attacks/augly/test_blur.py | 4 +- tests/attacks/targets/__init__.py | 7 --- tests/{ => counterfit}/core/test_core.py | 27 +++++----- .../core/test_state.py => mocks/__init__.py} | 0 tests/mocks/targets/__init__.py | 7 +++ .../{attacks => mocks}/targets/creditfraud.py | 0 .../targets/creditfraud/creditcard.npz | Bin .../creditfraud_sklearn_pipeline.pkl | Bin .../targets/digits_keras.py | 0 .../targets/digits_keras/mnist_784.npz | Bin .../targets/digits_keras/mnist_model.h5 | Bin .../{attacks => mocks}/targets/digits_mlp.py | 0 .../targets/digits_mlp/mnist_784.npz | Bin .../digits_mlp/mnist_sklearn_pipeline.pkl | Bin .../targets/movie_reviews.py | 0 .../movie-reviews-scores-full.csv | 0 .../movie_reviews/movie-reviews-scores.csv | 0 .../movie_reviews/movie-reviews-vocab.pkl | Bin .../movie_reviews_sentiment_analysis.pt | Bin .../movie_reviews_torch_model_params.pt | Bin tests/{attacks => mocks}/targets/satellite.py | 0 ...satellite-image-params-airplane-stadium.h5 | Bin ...tellite_images_airplane_stadium_196608.npz | Bin tests/test_art.py | 2 - tests/test_data/config.json | 33 +++++++++++++ tests/utils/__init__.py | 0 tests/utils/helpers.py | 46 ++++++++++++++++++ 35 files changed, 140 insertions(+), 53 deletions(-) create mode 100644 .coverage delete mode 100644 test_config.yml delete mode 100644 tests/attacks/targets/__init__.py rename tests/{ => counterfit}/core/test_core.py (60%) rename tests/{counterfit/core/test_state.py => mocks/__init__.py} (100%) create mode 100644 tests/mocks/targets/__init__.py rename tests/{attacks => mocks}/targets/creditfraud.py (100%) rename tests/{attacks => mocks}/targets/creditfraud/creditcard.npz (100%) rename tests/{attacks => mocks}/targets/creditfraud/creditfraud_sklearn_pipeline.pkl (100%) rename tests/{attacks => mocks}/targets/digits_keras.py (100%) rename tests/{attacks => mocks}/targets/digits_keras/mnist_784.npz (100%) rename tests/{attacks => mocks}/targets/digits_keras/mnist_model.h5 (100%) rename tests/{attacks => mocks}/targets/digits_mlp.py (100%) rename tests/{attacks => mocks}/targets/digits_mlp/mnist_784.npz (100%) rename tests/{attacks => mocks}/targets/digits_mlp/mnist_sklearn_pipeline.pkl (100%) rename tests/{attacks => mocks}/targets/movie_reviews.py (100%) rename tests/{attacks => mocks}/targets/movie_reviews/movie-reviews-scores-full.csv (100%) rename tests/{attacks => mocks}/targets/movie_reviews/movie-reviews-scores.csv (100%) rename tests/{attacks => mocks}/targets/movie_reviews/movie-reviews-vocab.pkl (100%) rename tests/{attacks => mocks}/targets/movie_reviews/movie_reviews_sentiment_analysis.pt (100%) rename tests/{attacks => mocks}/targets/movie_reviews/movie_reviews_torch_model_params.pt (100%) rename tests/{attacks => mocks}/targets/satellite.py (100%) rename tests/{attacks => mocks}/targets/satellite/satellite-image-params-airplane-stadium.h5 (100%) rename tests/{attacks => mocks}/targets/satellite/satellite_images_airplane_stadium_196608.npz (100%) delete mode 100644 tests/test_art.py create mode 100644 tests/test_data/config.json create mode 100644 tests/utils/__init__.py create mode 100644 tests/utils/helpers.py diff --git a/.coverage b/.coverage new file mode 100644 index 0000000000000000000000000000000000000000..e0ba9d61930df9e0ebcdc09c3fc674d5a1722a2c GIT binary patch literal 53248 zcmeI5dvF`adB6|2!{K?i;1hiFNKqmo0wDMhB~c!Wimk?KoHl;OQ^SUjfby|7S>DZ}d zll^uNj{}95{7*8P?}o&;x3}MZyTAQ*_x5mixb2pkV_8jE)YAzyt4wlU&SK$a6@}wC z7yL``U)byr5DGgWEv9g_poXB-aT zh6IoR5_8DY4{2Y+lXAl1oZC;d&wg6gL zZ3(hSrDF*-y{a71Rs$?S-da{Wo;4yM6D_usq$!kY6g-hxMtR+Qi8kAX?lexqkAU~W-q-f)ShG96XSJSMfo7MA`ZMbbJv}LKjWzAVY zPFvK_$|Yk*b6TCVz^akd0}8DKc))hI=-b<3;lL}kuIP)b%TR5_ zXdLinbDtS2z);h)qbYhmA(Zr&>TR^JPzGf#uFsYG8uOTp1|0;(f*IKWYpMzidYW*BN(;zJvYMT&L&)(eYx?wcaV(ecj!bPdk|#}4)g zXvqr0XyH&oQ!}}AVIZkBNES3oqhz6vMJ{@cnbL1Ri?V9P|%c=1zec3HWy>@*q*K5P?x``W4%oE9NEPCvzmZnd%^KVoO} z6ENu0f4;29G^SMKk7=vH6*Uf{NlJ~Sq07P;Y|H=(O}L;!Vs%c>WtsodUFM#2?kSbm z$e^@eU=OMMEID*?p&_YK9IznPFv4*W;R+G^PYdcqjv=Vi|>*zH&0!U&Q(%qetqza9Eop<6*9kx-Kh`Du#IzDBMvp$EE_3jQ8xlYQNB{{S0VIF~kN^@u z0!RP}AOR$B?GfNDR*Rjk{{?arM;;<+awi0ELjp(u2_OL^fCP{L5!sQ0~{r{Bk6h~esUm?FkK1oL8pUW@EeChe63 z&-Xol>bcvq&HbP5XWR+*i0j{7&$zO#S(n%Os`DwQ?wodPIKJWdeaEeiCi_eF&)Gj} zm&AV&pA_#Ax7#k*{=)VNn@_j^hHygyNB{|34S_vVJa_EiOhnBl;^vE<##KzyRZw_b zT4UEXqcjAUJEtdkE>l%g&1T_RX(poPmg1|1-dsGF2EE-AJeRAlm(ChmiCARd@bwW_MyuGIqcs7Iq+ATLzLBkeZ#P8JI@_$OyBG)dxt8wcZFJSb(KBa zDmPmN)h(VfJy_YXt7{sLvU{MihgHxk_me4w=k&(ln2=8N^W2fzT4^n%r|F9>U_d&a zg?8N4SGO5VI_g|5uF`=o)XQ`8wN25dbR+O)40udFwK~yLw`gooF4_W}$!?xY)V5Jf zUNkh_Rkx{1N+xy(JHekxZEMhr6WTF7O@}d{XG{yGAe8xr4#>v+VX`rmWyV*xXPb&* z%u4JN3UGZp6rj>fVDh;ulufqxwAJk-W_6g^a@km%4yV(tb;)nhgaHop_O~5~x!%Rj^isMZ@7prZRIj~7((p%RbnGcr?LhVANZbxBC1&U#r=ay=h zi>1%c5@2CW;yJAW4R|9!Do25bJ&l>rQ|yHo)P%b+6LEbBUWh|YxEeE&KeGajpp)m+ z25Lbo56Q3J2$joZBxgPm zS=Qq4q9WSEf_$yajnt;hhpjw!M}-O-nVS+0Sa|MGg@kYy!6Hozrs??0@WN zwtvrllXJJ@n~qJ#qmJ9iY1>8H7i_oKI)sbD)5I_Tk6ji|ia!wV6?c%|C3juTX+Ne# z0!RP}aNJ)0Sfx`RW56?=02wnVjBJJV|I}_iQ(d$A`hRi{pKCzx%Ju)mZwTJtW;c+b2hsE_gA4-Gpqdif8>C$Zu{`% zfD?k@Sz(>O>Ksfb9XqD^qYdhtePw%<_O4Ltt*$aurP(T|Zt;|DRUEszrm_AXtm0u6 z^veBYU>C192*-p}ndFbu(JHNd^Xvcq`psa{DX#zf#`yU-I-xP#0cit$hyFXMmrm}R@@7boJl&}AN z)n)>d&t0KxvfW<4lbF>_>;Ja;;Km4_!Es2+p_*|so%p)Mosl>p5SA3tZrTZ zH`Vt?=EEg};1J&W9fc_s*8gOjU#e3sV zk$mg=U#OY+73+U~gukOwg^kQjrLYe0hnyA8MNR8}eE%P=uaE!|Kmter2_OL^fCP{L z5QjlHd7YY%^nZPQqU691K~Kfj&houd%nI(qWKGa=Vb z2)6Hh?t{zEiNp_K#ee?D8<)o*t;M@*1R~-izu0{HCI4e1kavKya$cI1gKOgGkHvEe z{V;-XjBZ}5cNo+LhadRPr!KV97^@)efEam)=-p0_Y=`1M|CJY_8}74hJwp&PGj!?n zrqyb3(CjQ0j%zV8R=nG2`fb1co*su-*sgO8LQMPM$<4n%Y;g_H@WAwsFP!NT6$s19 z1KxhB(SH_RgT%Hqi?@%)^qu#dGh(=~yBA_QdRNx|;l6+Dkb7XKujkT>fDg9W&TiPB z>V7Zvgk{aJd5*@zCfd8;khklSg>*td={);g&#b^H%Ai;5fC#z6+Uf(j)3ujBUjMjt?Q4n7pV_$kh}c3iZLx~YR5t%qY@)Ks>hOYW z_MUvQ(?=i_Chw1NYYNn4_V*lXj%H5bluj8AOvxK3Pw2b|)wcX>^I>ab&HcBW;FjP} zhxFFk0bZ0O5A5`K7XRzQyFVBD?vmVe*Zu8BUk}W#?P+Rp!S0Cbt4r_OF1~c?rRy^P zcHXM=J0Wt``N00pi}##)XLJ9Td%5pDv}WI!<=2M(b>Q(KUUWbL*9$G7~gR2EE%HNXD$bwlOg{$i%<^6NBGdxF3O4;L$duGK2QH30VgKj$ri#{d#Q0!RP}AOR$R1dsp{Kmter2_OL^ za19cm-~Z$K{~ByqtP&DH0!RP}AOR$R1dsp{Kmter2_OLz0s8zuuK!IyFo*<@01`j~ WNB{{S0VIF~kN^@u0!ZK*B=G+)*H9w> literal 0 HcmV?d00001 diff --git a/.gitignore b/.gitignore index b0ce2e9..569b57e 100644 --- a/.gitignore +++ b/.gitignore @@ -46,7 +46,6 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ -.coverage .coverage.* .cache nosetests.xml diff --git a/conftest.py b/conftest.py index e69de29..48eced0 100644 --- a/conftest.py +++ b/conftest.py @@ -0,0 +1,28 @@ +from collections import OrderedDict, namedtuple +import json +import os +import pytest +import tests.utils.helpers as hp + +@pytest.fixture(scope='function', autouse=True) +def test_data(request, load_module_test_data): + """gets only required portion from module test data dictionary as test object""" + return hp.extract_test_data(request.node.originalname, load_module_test_data) + +@pytest.fixture(scope='module', autouse=True) +def load_module_test_data(request, load_session_test_data): + """gets only required portion from session test data dictionary""" + key = request.module.__name__ + if key not in load_session_test_data: + return {} + return load_session_test_data[request.module.__name__] + +@pytest.fixture(scope='session', autouse=True) +def load_session_test_data(): + """loads test data from json as dictionary""" + folder_path = os.path.abspath(os.path.dirname(__file__)) + folder = os.path.join(folder_path, *["tests", "test_data"]) + file = os.path.join(folder, "config.json") + with open(file) as fp: + data = json.load(fp) + return data \ No newline at end of file diff --git a/test_config.yml b/test_config.yml deleted file mode 100644 index 1e8d81d..0000000 --- a/test_config.yml +++ /dev/null @@ -1,5 +0,0 @@ -target: - creditfraud -attacks: - attack_1: - attack_name: hop_skip_jump \ No newline at end of file diff --git a/tests/attacks/art/test_blackbox_rule_based.py b/tests/attacks/art/test_blackbox_rule_based.py index 59351f2..aec8ecc 100644 --- a/tests/attacks/art/test_blackbox_rule_based.py +++ b/tests/attacks/art/test_blackbox_rule_based.py @@ -12,13 +12,12 @@ from counterfit.targets import CreditFraud, Digits, DigitKeras, SatelliteImages from counterfit import Counterfit - @pytest.fixture(params=[CreditFraud, Digits, DigitKeras, SatelliteImages]) def target(request): yield request.param # Evasion -@pytest.fixture(params=['boundary', 'hop_skip_jump']) +@pytest.fixture(params=['hop_skip_jump']) def attack(request): yield request.param @@ -29,16 +28,6 @@ def build_attack(target_obj: 'CFTarget', attack: str): return Counterfit.build_attack(target, attack) -def test_boundary_credit(): - attack = build_attack(CreditFraud, 'boundary') - # attack = build_attack(CreditFraud, 'hop_skip_jump') - # >>>> run() attack estimator: BlackBoxClassifier(model=None, clip_values=None, preprocessing=StandardisationMeanStd(mean=0.0, std=1.0, apply_fit=True, apply_predict=True), preprocessing_defences=None, postprocessing_defences=None, preprocessing_operations=[StandardisationMeanStd(mean=0.0, std=1.0, apply_fit=True, apply_predict=True)], nb_classes=2, predict_fn=>, input_shape=(30,)) - attack_did_succeed = Counterfit.run_attack(attack) - - assert attack - assert attack_did_succeed - - def test_attack(target, attack): cfattack = build_attack(target, attack) assert cfattack diff --git a/tests/attacks/art/test_hop_skip_jump.py b/tests/attacks/art/test_hop_skip_jump.py index 8b8a2e0..f1c36e3 100644 --- a/tests/attacks/art/test_hop_skip_jump.py +++ b/tests/attacks/art/test_hop_skip_jump.py @@ -10,15 +10,15 @@ warnings.filterwarnings("ignore", category=FutureWarning) warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.filterwarnings("ignore", category=UserWarning) -from targets import CreditFraud -from targets import Digits -from targets import DigitKeras -from targets import SatelliteImages +from tests.mocks.targets import CreditFraud +from tests.mocks.targets import Digits +from tests.mocks.targets import DigitKeras +from tests.mocks.targets import SatelliteImages from counterfit import Counterfit -@pytest.fixture(params=[CreditFraud, Digits, DigitKeras]) +@pytest.fixture(params=[CreditFraud, DigitKeras]) def target(request): yield request.param diff --git a/tests/attacks/art/test_label_only_boundary_distance.py b/tests/attacks/art/test_label_only_boundary_distance.py index 4aa96b0..e73de89 100644 --- a/tests/attacks/art/test_label_only_boundary_distance.py +++ b/tests/attacks/art/test_label_only_boundary_distance.py @@ -8,13 +8,13 @@ warnings.filterwarnings("ignore", category=FutureWarning) warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.filterwarnings("ignore", category=UserWarning) -from targets import CreditFraud -from targets import Digits -from targets import DigitKeras +from tests.mocks.targets import CreditFraud +from tests.mocks.targets import Digits +from tests.mocks.targets import DigitKeras from counterfit import Counterfit -@pytest.fixture(params=[CreditFraud, Digits, DigitKeras]) +@pytest.fixture(params=[DigitKeras]) def target(request): yield request.param diff --git a/tests/attacks/art/test_mi_face.py b/tests/attacks/art/test_mi_face.py index 5284f5f..fc6e57e 100644 --- a/tests/attacks/art/test_mi_face.py +++ b/tests/attacks/art/test_mi_face.py @@ -8,7 +8,7 @@ warnings.filterwarnings("ignore", category=FutureWarning) warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.filterwarnings("ignore", category=UserWarning) -from targets import DigitKeras +from tests.mocks.targets import DigitKeras from counterfit import Counterfit diff --git a/tests/attacks/augly/test_blur.py b/tests/attacks/augly/test_blur.py index 8cff02a..60442f3 100644 --- a/tests/attacks/augly/test_blur.py +++ b/tests/attacks/augly/test_blur.py @@ -5,8 +5,8 @@ warnings.filterwarnings('ignore') sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) -from targets import Digits -from targets import DigitKeras +from tests.mocks.targets import Digits +from tests.mocks.targets import DigitKeras from counterfit import Counterfit diff --git a/tests/attacks/targets/__init__.py b/tests/attacks/targets/__init__.py deleted file mode 100644 index 8605ec3..0000000 --- a/tests/attacks/targets/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from counterfit.core.targets import CFTarget - -from .creditfraud import CreditFraud -from .digits_mlp import Digits -from .digits_keras import DigitKeras -from .movie_reviews import MovieReviewsTarget -from .satellite import SatelliteImages \ No newline at end of file diff --git a/tests/core/test_core.py b/tests/counterfit/core/test_core.py similarity index 60% rename from tests/core/test_core.py rename to tests/counterfit/core/test_core.py index a4fb264..5e73841 100644 --- a/tests/core/test_core.py +++ b/tests/counterfit/core/test_core.py @@ -15,36 +15,35 @@ def test_frameworks(): frameworks = Counterfit.get_frameworks() assert dict(frameworks) -def test_build_target(): +def test_build_target(test_data): target = Counterfit.build_target( - data_type="images", - endpoint="http://locahost/score", - output_classes=["Cat", "NotACat"], - classifier="closed-box", + data_type=test_data.data_type, + endpoint=test_data.endpoint, + output_classes=test_data.output_classes, + classifier=test_data.classifier, input_shape=(1,), load_func=load, predict_func=predict, - X = [[1,0]] + X = test_data.X ) - assert isinstance(target, CFTarget) -def test_build_attack(): +def test_build_attack(test_data): target = Counterfit.build_target( - data_type="image", - endpoint="http://locahost/score", - output_classes=["Cat", "NotACat"], - classifier="closed-box", + data_type=test_data.data_type, + endpoint=test_data.endpoint, + output_classes=test_data.output_classes, + classifier=test_data.classifier, input_shape=(1,), load_func=load, predict_func=predict, - X = [[1,0]] + X = test_data.X ) cfattack = Counterfit.build_attack( target=target, - attack="hop_skip_jump" + attack=test_data.attack ) assert isinstance(cfattack, CFAttack) diff --git a/tests/counterfit/core/test_state.py b/tests/mocks/__init__.py similarity index 100% rename from tests/counterfit/core/test_state.py rename to tests/mocks/__init__.py diff --git a/tests/mocks/targets/__init__.py b/tests/mocks/targets/__init__.py new file mode 100644 index 0000000..260909e --- /dev/null +++ b/tests/mocks/targets/__init__.py @@ -0,0 +1,7 @@ +from counterfit.core.targets import CFTarget + +from tests.mocks.targets.creditfraud import CreditFraud +from tests.mocks.targets.digits_mlp import Digits +from tests.mocks.targets.digits_keras import DigitKeras +from tests.mocks.targets.movie_reviews import MovieReviewsTarget +from tests.mocks.targets.satellite import SatelliteImages \ No newline at end of file diff --git a/tests/attacks/targets/creditfraud.py b/tests/mocks/targets/creditfraud.py similarity index 100% rename from tests/attacks/targets/creditfraud.py rename to tests/mocks/targets/creditfraud.py diff --git a/tests/attacks/targets/creditfraud/creditcard.npz b/tests/mocks/targets/creditfraud/creditcard.npz similarity index 100% rename from tests/attacks/targets/creditfraud/creditcard.npz rename to tests/mocks/targets/creditfraud/creditcard.npz diff --git a/tests/attacks/targets/creditfraud/creditfraud_sklearn_pipeline.pkl b/tests/mocks/targets/creditfraud/creditfraud_sklearn_pipeline.pkl similarity index 100% rename from tests/attacks/targets/creditfraud/creditfraud_sklearn_pipeline.pkl rename to tests/mocks/targets/creditfraud/creditfraud_sklearn_pipeline.pkl diff --git a/tests/attacks/targets/digits_keras.py b/tests/mocks/targets/digits_keras.py similarity index 100% rename from tests/attacks/targets/digits_keras.py rename to tests/mocks/targets/digits_keras.py diff --git a/tests/attacks/targets/digits_keras/mnist_784.npz b/tests/mocks/targets/digits_keras/mnist_784.npz similarity index 100% rename from tests/attacks/targets/digits_keras/mnist_784.npz rename to tests/mocks/targets/digits_keras/mnist_784.npz diff --git a/tests/attacks/targets/digits_keras/mnist_model.h5 b/tests/mocks/targets/digits_keras/mnist_model.h5 similarity index 100% rename from tests/attacks/targets/digits_keras/mnist_model.h5 rename to tests/mocks/targets/digits_keras/mnist_model.h5 diff --git a/tests/attacks/targets/digits_mlp.py b/tests/mocks/targets/digits_mlp.py similarity index 100% rename from tests/attacks/targets/digits_mlp.py rename to tests/mocks/targets/digits_mlp.py diff --git a/tests/attacks/targets/digits_mlp/mnist_784.npz b/tests/mocks/targets/digits_mlp/mnist_784.npz similarity index 100% rename from tests/attacks/targets/digits_mlp/mnist_784.npz rename to tests/mocks/targets/digits_mlp/mnist_784.npz diff --git a/tests/attacks/targets/digits_mlp/mnist_sklearn_pipeline.pkl b/tests/mocks/targets/digits_mlp/mnist_sklearn_pipeline.pkl similarity index 100% rename from tests/attacks/targets/digits_mlp/mnist_sklearn_pipeline.pkl rename to tests/mocks/targets/digits_mlp/mnist_sklearn_pipeline.pkl diff --git a/tests/attacks/targets/movie_reviews.py b/tests/mocks/targets/movie_reviews.py similarity index 100% rename from tests/attacks/targets/movie_reviews.py rename to tests/mocks/targets/movie_reviews.py diff --git a/tests/attacks/targets/movie_reviews/movie-reviews-scores-full.csv b/tests/mocks/targets/movie_reviews/movie-reviews-scores-full.csv similarity index 100% rename from tests/attacks/targets/movie_reviews/movie-reviews-scores-full.csv rename to tests/mocks/targets/movie_reviews/movie-reviews-scores-full.csv diff --git a/tests/attacks/targets/movie_reviews/movie-reviews-scores.csv b/tests/mocks/targets/movie_reviews/movie-reviews-scores.csv similarity index 100% rename from tests/attacks/targets/movie_reviews/movie-reviews-scores.csv rename to tests/mocks/targets/movie_reviews/movie-reviews-scores.csv diff --git a/tests/attacks/targets/movie_reviews/movie-reviews-vocab.pkl b/tests/mocks/targets/movie_reviews/movie-reviews-vocab.pkl similarity index 100% rename from tests/attacks/targets/movie_reviews/movie-reviews-vocab.pkl rename to tests/mocks/targets/movie_reviews/movie-reviews-vocab.pkl diff --git a/tests/attacks/targets/movie_reviews/movie_reviews_sentiment_analysis.pt b/tests/mocks/targets/movie_reviews/movie_reviews_sentiment_analysis.pt similarity index 100% rename from tests/attacks/targets/movie_reviews/movie_reviews_sentiment_analysis.pt rename to tests/mocks/targets/movie_reviews/movie_reviews_sentiment_analysis.pt diff --git a/tests/attacks/targets/movie_reviews/movie_reviews_torch_model_params.pt b/tests/mocks/targets/movie_reviews/movie_reviews_torch_model_params.pt similarity index 100% rename from tests/attacks/targets/movie_reviews/movie_reviews_torch_model_params.pt rename to tests/mocks/targets/movie_reviews/movie_reviews_torch_model_params.pt diff --git a/tests/attacks/targets/satellite.py b/tests/mocks/targets/satellite.py similarity index 100% rename from tests/attacks/targets/satellite.py rename to tests/mocks/targets/satellite.py diff --git a/tests/attacks/targets/satellite/satellite-image-params-airplane-stadium.h5 b/tests/mocks/targets/satellite/satellite-image-params-airplane-stadium.h5 similarity index 100% rename from tests/attacks/targets/satellite/satellite-image-params-airplane-stadium.h5 rename to tests/mocks/targets/satellite/satellite-image-params-airplane-stadium.h5 diff --git a/tests/attacks/targets/satellite/satellite_images_airplane_stadium_196608.npz b/tests/mocks/targets/satellite/satellite_images_airplane_stadium_196608.npz similarity index 100% rename from tests/attacks/targets/satellite/satellite_images_airplane_stadium_196608.npz rename to tests/mocks/targets/satellite/satellite_images_airplane_stadium_196608.npz diff --git a/tests/test_art.py b/tests/test_art.py deleted file mode 100644 index 8f69665..0000000 --- a/tests/test_art.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_run(): - assert False diff --git a/tests/test_data/config.json b/tests/test_data/config.json new file mode 100644 index 0000000..b16adc8 --- /dev/null +++ b/tests/test_data/config.json @@ -0,0 +1,33 @@ +{ + "test_core": { + "test_build_target": { + "data_type": "images", + "endpoint": "http://locahost/score", + "output_classes": ["Cat", "NotACat"], + "classifier": "closed-box", + "X": [[1,0]] + }, + "test_build_attack": { + "data_type": "image", + "endpoint": "http://locahost/score", + "output_classes": ["Cat", "NotACat"], + "classifier": "closed-box", + "X": [[1,0]], + "attack": "hop_skip_jump" + } + }, + "test_hop_skip_jump": { + "test_attack": { + "attack_targets": [ + { + "attack": "hop_skip_jump", + "target": "CreditFraud" + }, + { + "attack": "hop_skip_jump", + "target": "DigitKeras" + } + ] + } + } +} \ No newline at end of file diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/utils/helpers.py b/tests/utils/helpers.py new file mode 100644 index 0000000..00af69f --- /dev/null +++ b/tests/utils/helpers.py @@ -0,0 +1,46 @@ +from collections import OrderedDict, namedtuple +from tests.mocks.targets import CreditFraud +from tests.mocks.targets import Digits +from tests.mocks.targets import DigitKeras +from tests.mocks.targets import SatelliteImages + +def get_target_class(target: str): + """loads attack-target combinations from json as dictionary""" + target_classes = { + "CreditFraud": CreditFraud(), + "Digits": Digits(), + "DigitKeras": DigitKeras(), + "SatelliteImages": SatelliteImages() + } + return target_classes[target] + +def extract_test_data(key: str, load_module_test_data): + """gets only required portion from module test data dictionary as test object""" + if key not in load_module_test_data: + return + return create_namedtuple_from_dict(load_module_test_data[key]) + + +def create_namedtuple_from_dict(obj): + """converts given list or dict to named tuples, generic alternative to dataclass""" + if isinstance(obj, dict): + fields = sorted(obj.keys()) + namedtuple_type = namedtuple( + typename='test_data', + field_names=fields, + rename=True, + ) + field_value_pairs = OrderedDict( + (str(field), create_namedtuple_from_dict(obj[field])) + for field in fields + ) + try: + return namedtuple_type(**field_value_pairs) + except TypeError: + # Cannot create namedtuple instance so fallback to dict (invalid attribute names) + return dict(**field_value_pairs) + elif isinstance(obj, (list, set, tuple, frozenset)): + return [create_namedtuple_from_dict(item) for item in obj] + else: + return obj +