diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 50ddad82..2e92e48e 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -29,19 +29,20 @@ jobs: matrix: test-groups: [ - "test/test_[a-e]*", - "test/test_[f-h]*", - "test/test_[i-o,q-r,t-z]*", - "test/test_[p]*", - "test/test_[s]*", - "test/storage/*", - "test/extension/*", + 0, + 1, + 2, + 3, + 4, + 5, + 6, + "other" ] fail-fast: false steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: ./scripts/ci.sh + - run: python ./scripts/ci.py ./scripts/distribution.json env: DISPLAY: ":99.0" TESTS: ${{ matrix.test-groups }} diff --git a/.gitignore b/.gitignore index b9bf1801..f471b8d7 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,5 @@ docs/apidoc/ node_modules datadir + +junit-report.xml \ No newline at end of file diff --git a/scripts/ci.py b/scripts/ci.py new file mode 100644 index 00000000..280d8300 --- /dev/null +++ b/scripts/ci.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# Usage: TESTS=0 python scripts/ci.py scripts/distribution.json +import json +import os +import subprocess +from sys import argv + +test_selection = os.getenv("TESTS") +assert len(argv) == 2 +if test_selection is None: + print("Please set TESTS environment variable") + exit(1) +with open(argv[1]) as f: + test_distribution = json.load(f) +if test_selection == "other": + res = subprocess.run("pytest --collect-only -q", capture_output=True, shell=True) + res.check_returncode() + actual_tests = res.stdout.decode("utf-8").splitlines() + for index, test in enumerate(actual_tests): + if len(test) == 0: # cut off warnings + actual_tests = actual_tests[:index] + + all_known_tests = set(sum(test_distribution, [])) + actual_tests_set = set(actual_tests) + known_but_dont_exist = all_known_tests.difference(actual_tests_set) + exist_but_arent_known = actual_tests_set.difference(all_known_tests) + if len(known_but_dont_exist) > 0 or len(exist_but_arent_known) > 0: + print("known_but_dont_exist:", known_but_dont_exist) + print("exist_but_arent_known", exist_but_arent_known) + print("Uncovered or outdated tests") + exit(2) +else: + index = int(test_selection) + tests = " ".join('"' + test + '"' for test in test_distribution[index]) + subprocess.run( + "pytest " + "--cov=openwpm --junit-xml=junit-report.xml " + f"--cov-report=xml {tests} " + "-s -v --durations=10;", + shell=True, + check=True, + ) + subprocess.run("codecov -f coverage.xml", shell=True, check=True) diff --git a/scripts/ci.sh b/scripts/ci.sh deleted file mode 100755 index 75f56930..00000000 --- a/scripts/ci.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -python -m pytest --cov=openwpm --junit-xml=junit-report.xml --cov-report=xml $TESTS -s -v --durations=10; -exit_code=$?; -if [[ "$exit_code" -ne 0 ]]; then - exit $exit_code; -fi -codecov -f coverage.xml; - diff --git a/scripts/distribute_tests.py b/scripts/distribute_tests.py new file mode 100644 index 00000000..74531dde --- /dev/null +++ b/scripts/distribute_tests.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# Usage: python scripts/distribute_tests.py junit-report.xml 7 scripts/distribution.json +# Where junit-report.xml is the result of running +# python -m pytest --cov=openwpm --junit-xml=junit-report.xml --cov-report=xml test/ -s -v --durations=10 +import json +from sys import argv +from typing import Any +from xml.etree import ElementTree as ET + +assert len(argv) == 4 + +num_runner = int(argv[2]) +tree = ET.parse(argv[1]) +root = tree.getroot() +testcases: list[dict[str, Any]] = [] +for testcase in root.iter("testcase"): + # Build correct test name based on naming convention + classname = testcase.get("classname") + assert isinstance(classname, str) + split = classname.rsplit(".", 1) + if split[1][0].isupper(): # Test is in a class + split[0] = split[0].replace(".", "/") + split[0] = split[0] + ".py" + path = split[0] + "::" + split[1] + else: + path = classname.replace(".", "/") + ".py" + time = testcase.get("time") + assert isinstance(time, str) + testcases.append({"path": f'{path}::{testcase.get("name")}', "time": float(time)}) + +sorted_testcases = sorted(testcases, key=lambda x: x["time"], reverse=True) +total_time = sum(k["time"] + 0.5 for k in sorted_testcases) +time_per_runner = total_time / num_runner +print(f"Total time: {total_time} Total time per runner: {total_time / num_runner}") +distributed_testcases: list[list[str]] = [[] for _ in range(num_runner)] +estimated_time = [] +for subsection in distributed_testcases: + time_spent = 0 + tmp = [] + for testcase in sorted_testcases: + if time_spent + testcase["time"] < time_per_runner: + tmp.append(testcase) + time_spent += testcase["time"] + 0.5 # account for overhead per testcase + for testcase in tmp: + sorted_testcases.remove(testcase) + subsection[:] = [testcase["path"] for testcase in tmp] + estimated_time.append(time_spent) + +assert len(sorted_testcases) == 0, print(len(sorted_testcases)) +print(estimated_time) +with open(argv[3], "w") as f: + json.dump(distributed_testcases, f, indent=0) diff --git a/scripts/distribution.json b/scripts/distribution.json new file mode 100644 index 00000000..9f2bb9db --- /dev/null +++ b/scripts/distribution.json @@ -0,0 +1,142 @@ +[ +[ +"test/test_profile.py::test_profile_recovery[on_crash_during_launch-stateful-without_seed_tar]", +"test/test_profile.py::test_profile_recovery[on_crash_during_launch-stateless-with_seed_tar]", +"test/test_profile.py::test_profile_recovery[on_crash_during_launch-stateful-with_seed_tar]", +"test/test_profile.py::test_profile_saved_when_launch_crashes", +"test/test_simple_commands.py::test_save_screenshot_valid[headless]", +"test/test_profile.py::test_load_tar_file", +"test/test_js_instrument_py.py::test_validate_bad__log_settings_missing", +"test/test_js_instrument_py.py::test_validate_good" +], +[ +"test/test_profile.py::test_profile_recovery[on_crash-stateless-with_seed_tar]", +"test/test_profile.py::test_profile_recovery[on_crash-stateful-with_seed_tar]", +"test/test_profile.py::test_profile_recovery[on_timeout-stateful-with_seed_tar]", +"test/test_profile.py::test_profile_recovery[on_normal_operation-stateless-with_seed_tar]", +"test/test_profile.py::test_profile_recovery[on_crash-stateful-without_seed_tar]", +"test/test_js_instrument_py.py::test_validate_bad__log_settings_invalid", +"test/test_js_instrument_py.py::test_api_collection_fingerprinting", +"test/test_js_instrument_py.py::test_validate_bad__not_a_list", +"test/test_js_instrument_py.py::test_validate_bad__missing_object", +"test/test_js_instrument_py.py::test_validated_bad__missing_instrumentedName", +"test/test_js_instrument_py.py::test_merge_and_validate_multiple_overlap_properties_to_instrument_properties_to_exclude", +"test/storage/test_storage_providers.py::test_basic_access[memory_structured]", +"test/storage/test_storage_providers.py::test_basic_access[memory_arrow]", +"test/storage/test_storage_providers.py::test_basic_access[sqlite]", +"test/test_js_instrument_py.py::test_complete_pass", +"test/storage/test_storage_providers.py::test_basic_unstructured_storing[memory_unstructured]", +"test/test_task_manager.py::test_failure_limit_value" +], +[ +"test/test_profile.py::test_profile_recovery[on_timeout-stateful-without_seed_tar]", +"test/test_profile.py::test_profile_recovery[on_timeout-stateless-with_seed_tar]", +"test/test_profile.py::test_profile_recovery[on_normal_operation-stateful-with_seed_tar]", +"test/test_profile.py::test_dump_profile_command", +"test/test_profile.py::test_profile_recovery[on_normal_operation-stateful-without_seed_tar]", +"test/test_extension.py::TestExtension::test_extension_gets_correct_visit_id", +"test/storage/test_storage_providers.py::test_basic_unstructured_storing[leveldb]", +"test/storage/test_storage_providers.py::test_basic_unstructured_storing[local_gzip]", +"test/test_callstack_instrument.py::test_http_stacktrace", +"test/test_dataclass_validations.py::test_display_mode", +"test/test_dataclass_validations.py::test_browser_type", +"test/test_dataclass_validations.py::test_tp_cookies_opt", +"test/test_dataclass_validations.py::test_save_content_type", +"test/test_dataclass_validations.py::test_log_file_extension", +"test/test_dataclass_validations.py::test_failure_limit", +"test/test_dataclass_validations.py::test_num_browser_crawl_config" +], +[ +"test/test_task_manager.py::test_assertion_error_propagation[False-expectation0]", +"test/test_xvfb_browser.py::test_display_shutdown", +"test/extension/test_startup_timeout.py::test_extension_startup_timeout", +"test/test_simple_commands.py::test_browse_wrapper_http_table_valid[headless]", +"test/test_task_manager.py::test_assertion_error_propagation[True-expectation1]", +"test/test_simple_commands.py::test_browse_http_table_valid[headless]", +"test/test_simple_commands.py::test_browse_http_table_valid[xvfb]", +"test/test_simple_commands.py::test_browse_wrapper_http_table_valid[xvfb]", +"test/test_extension.py::test_audio_fingerprinting", +"test/test_profile.py::test_profile_error", +"test/test_http_instrumentation.py::TestHTTPInstrument::test_service_worker_requests", +"test/test_js_instrument_py.py::test_merge_and_validate_multiple_overlap_properties", +"test/test_js_instrument_py.py::test_merge_when_log_settings_is_null", +"test/test_webdriver_utils.py::test_parse_neterror" +], +[ +"test/test_http_instrumentation.py::test_cache_hits_recorded", +"test/test_task_manager.py::test_failure_limit_reset", +"test/test_profile.py::test_crash_during_init", +"test/test_simple_commands.py::test_dump_page_source_valid[headless]", +"test/test_profile.py::test_saving", +"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_x_www_form_urlencoded", +"test/test_js_instrument.py::TestJSInstrumentRecursiveProperties::test_instrument_object", +"test/test_simple_commands.py::test_save_screenshot_valid[xvfb]", +"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_text_plain", +"test/test_http_instrumentation.py::test_javascript_saving", +"test/test_http_instrumentation.py::test_content_saving", +"test/test_http_instrumentation.py::TestHTTPInstrument::test_worker_script_requests", +"test/test_extension.py::TestExtension::test_js_call_stack", +"test/test_profile.py::test_crash_profile", +"test/test_extension.py::TestExtension::test_canvas_fingerprinting", +"test/test_crawl.py::test_browser_profile_coverage", +"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_file_upload", +"test/test_js_instrument_py.py::test_merge_diff_instrumented_names", +"test/test_js_instrument_py.py::test_merge_multiple_duped_properties", +"test/test_js_instrument_py.py::test_merge_multiple_duped_properties_different_log_settings", +"test/test_js_instrument_py.py::test_api_whole_module", +"test/test_js_instrument_py.py::test_api_two_keys_in_shortcut", +"test/test_js_instrument_py.py::test_api_instances_on_window", +"test/test_js_instrument_py.py::test_api_instances_on_window_with_properties", +"test/test_js_instrument_py.py::test_api_module_specific_properties", +"test/test_js_instrument_py.py::test_api_passing_partial_log_settings" +], +[ +"test/test_simple_commands.py::test_recursive_dump_page_source_valid[xvfb]", +"test/extension/test_logging.py::test_extension_logging", +"test/test_timer.py::test_command_duration", +"test/test_http_instrumentation.py::test_page_visit[True]", +"test/storage/test_storage_controller.py::test_arrow_provider", +"test/storage/test_storage_controller.py::test_startup_and_shutdown", +"test/test_mp_logger.py::test_multiple_instances", +"test/test_simple_commands.py::test_recursive_dump_page_source_valid[headless]", +"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_ajax_no_key_value", +"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_formdata", +"test/test_js_instrument.py::TestJSInstrumentExistingWindowProperty::test_instrument_object", +"test/test_simple_commands.py::test_get_http_tables_valid[xvfb]", +"test/test_simple_commands.py::test_dump_page_source_valid[xvfb]", +"test/test_js_instrument.py::TestJSInstrument::test_instrument_object", +"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_ajax", +"test/test_dns_instrument.py::test_name_resolution", +"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_binary_post_data", +"test/test_http_instrumentation.py::test_document_saving", +"test/test_storage_vectors.py::test_js_profile_cookies", +"test/test_extension.py::TestExtension::test_js_time_stamp", +"test/storage/test_storage_providers.py::test_local_arrow_storage_provider" +], +[ +"test/test_js_instrument.py::TestJSInstrumentMockWindowProperty::test_instrument_object", +"test/test_extension.py::TestExtension::test_document_cookie_instrumentation", +"test/test_custom_function_command.py::test_custom_function", +"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_ajax_no_key_value_base64_encoded", +"test/test_extension.py::TestExtension::test_property_enumeration", +"test/test_simple_commands.py::test_get_site_visits_table_valid[xvfb]", +"test/test_js_instrument.py::TestJSInstrumentNonExistingWindowProperty::test_instrument_object", +"test/test_simple_commands.py::test_get_http_tables_valid[headless]", +"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_multipart_formdata", +"test/test_http_instrumentation.py::test_page_visit[False]", +"test/test_js_instrument.py::TestJSInstrumentByPython::test_instrument_object", +"test/test_profile.py::test_seed_persistence", +"test/test_callback.py::test_local_callbacks", +"test/test_simple_commands.py::test_get_site_visits_table_valid[headless]", +"test/test_simple_commands.py::test_browse_site_visits_table_valid[headless]", +"test/test_task_manager.py::test_failure_limit_exceeded", +"test/test_extension.py::TestExtension::test_webrtc_localip", +"test/test_simple_commands.py::test_browse_site_visits_table_valid[xvfb]", +"test/storage/test_arrow_cache.py::test_arrow_cache", +"test/test_profile.py::test_save_incomplete_profile_error", +"test/test_webdriver_utils.py::test_parse_neterror_integration", +"test/test_mp_logger.py::test_multiprocess", +"test/test_mp_logger.py::test_child_process_with_exception", +"test/test_mp_logger.py::test_child_process_logging" +] +] \ No newline at end of file