diff --git a/docker-compose.private.yml b/docker-compose.private.yml index fe4abc8953..a0dbc2ebd0 100644 --- a/docker-compose.private.yml +++ b/docker-compose.private.yml @@ -4,7 +4,6 @@ services: worker: depends_on: - customs - - wat customs: build: @@ -16,14 +15,3 @@ services: - "nginx:olympia.test" depends_on: - web - - wat: - build: - context: ./private/addons-wat - environment: - - ALLOWED_ORIGIN=http://olympia.test - - PORT=10102 - links: - - "nginx:olympia.test" - depends_on: - - web diff --git a/private/README.md b/private/README.md index f35d6c97ca..e28435429b 100644 --- a/private/README.md +++ b/private/README.md @@ -7,7 +7,6 @@ This folder should contain some of our private tools. In the following, we descr Make sure to clone: 1. the `customs` repo in `private/addons-customs-scanner` -2. the `wat` repo in `private/addons-wat` Specify the [`docker-compose.private.yml`](../docker-compose.private.yml) file to `docker-compose` (together with the default [`docker-compose.yml`](../docker-compose.yml) file) to build the Docker images: @@ -30,15 +29,6 @@ $ make shell $ [root@:/code#] ./manage.py waffle_switch enable-customs on ``` -### wat - -A waffle switch is used to enable/disable the `wat` Celery task: - -``` -$ make shell -$ [root@:/code#] ./manage.py waffle_switch enable-wat on -``` - ## Yara If you have access to `yara`, you should first clone it in `private/addons-yara`. diff --git a/settings.py b/settings.py index fdc1c9d7e1..642914d48e 100644 --- a/settings.py +++ b/settings.py @@ -152,8 +152,6 @@ INBOUND_EMAIL_VALIDATION_KEY = 'totally-unsecure-validation-string' # Sectools CUSTOMS_API_URL = 'http://customs:10101/' CUSTOMS_API_KEY = 'customssecret' -WAT_API_URL = 'http://wat:10102/' -WAT_API_KEY = 'watsecret' REMOTE_SETTINGS_IS_TEST_SERVER = True diff --git a/src/olympia/constants/scanners.py b/src/olympia/constants/scanners.py index 9993477c7c..bbb2088ca9 100644 --- a/src/olympia/constants/scanners.py +++ b/src/olympia/constants/scanners.py @@ -2,11 +2,14 @@ from django.utils.translation import gettext_lazy as _ CUSTOMS = 1 -WAT = 2 +# We do not use the WAT scanner anymore but we keep this constant for the model +# definition. We shouldn't use this constant, though. +# See: https://github.com/mozilla/addons-server/issues/19152 +_WAT = 2 YARA = 3 MAD = 4 -SCANNERS = {CUSTOMS: 'customs', WAT: 'wat', YARA: 'yara', MAD: 'mad'} +SCANNERS = {CUSTOMS: 'customs', _WAT: 'wat', YARA: 'yara', MAD: 'mad'} # Action IDs are also used for severity (the higher, the more severe). # The field is a PositiveSmallIntegerField, it should go up to 65535. diff --git a/src/olympia/devhub/tests/test_utils.py b/src/olympia/devhub/tests/test_utils.py index 439001365f..5fda8368be 100644 --- a/src/olympia/devhub/tests/test_utils.py +++ b/src/olympia/devhub/tests/test_utils.py @@ -27,7 +27,7 @@ from olympia.applications.models import AppVersion from olympia.devhub import tasks, utils from olympia.files.tasks import repack_fileupload from olympia.files.tests.test_models import UploadMixin -from olympia.scanners.tasks import run_customs, run_wat, run_yara, call_mad_api +from olympia.scanners.tasks import run_customs, run_yara, call_mad_api from olympia.versions.models import Version @@ -439,49 +439,6 @@ class TestValidator(UploadMixin, TestCase): tasks.handle_upload_validation_result.s(file_upload.pk, channel, False), ) - @mock.patch('olympia.devhub.utils.chain') - def test_adds_run_wat_when_enabled(self, mock_chain): - self.create_switch('enable-wat', active=True) - file_upload = self.get_upload('webextension.xpi', with_validation=False) - channel = amo.RELEASE_CHANNEL_LISTED - - utils.Validator(file_upload, listed=True) - - mock_chain.assert_called_once_with( - tasks.create_initial_validation_results.si(), - repack_fileupload.s(file_upload.pk), - tasks.validate_upload.s(file_upload.pk, channel), - tasks.check_for_api_keys_in_file.s(file_upload.pk), - chord( - [ - tasks.forward_linter_results.s(file_upload.pk), - run_wat.s(file_upload.pk), - ], - call_mad_api.s(file_upload.pk), - ), - tasks.handle_upload_validation_result.s(file_upload.pk, channel, False), - ) - - @mock.patch('olympia.devhub.utils.chain') - def test_does_not_add_run_wat_when_disabled(self, mock_chain): - self.create_switch('enable-wat', active=False) - file_upload = self.get_upload('webextension.xpi', with_validation=False) - channel = amo.RELEASE_CHANNEL_LISTED - - utils.Validator(file_upload, listed=True) - - mock_chain.assert_called_once_with( - tasks.create_initial_validation_results.si(), - repack_fileupload.s(file_upload.pk), - tasks.validate_upload.s(file_upload.pk, channel), - tasks.check_for_api_keys_in_file.s(file_upload.pk), - chord( - [tasks.forward_linter_results.s(file_upload.pk)], - call_mad_api.s(file_upload.pk), - ), - tasks.handle_upload_validation_result.s(file_upload.pk, channel, False), - ) - @mock.patch('olympia.devhub.utils.chain') def test_adds_yara_and_customs(self, mock_chain): self.create_switch('enable-customs', active=True) @@ -510,7 +467,6 @@ class TestValidator(UploadMixin, TestCase): @mock.patch('olympia.devhub.utils.chain') def test_adds_all_scanners(self, mock_chain): self.create_switch('enable-customs', active=True) - self.create_switch('enable-wat', active=True) self.create_switch('enable-yara', active=True) file_upload = self.get_upload('webextension.xpi', with_validation=False) channel = amo.RELEASE_CHANNEL_LISTED @@ -527,7 +483,6 @@ class TestValidator(UploadMixin, TestCase): tasks.forward_linter_results.s(file_upload.pk), run_yara.s(file_upload.pk), run_customs.s(file_upload.pk), - run_wat.s(file_upload.pk), ], call_mad_api.s(file_upload.pk), ), @@ -536,7 +491,6 @@ class TestValidator(UploadMixin, TestCase): def test_create_file_upload_tasks(self): self.create_switch('enable-customs', active=True) - self.create_switch('enable-wat', active=True) self.create_switch('enable-yara', active=True) file_upload = self.get_upload('webextension.xpi', with_validation=False) channel = amo.RELEASE_CHANNEL_LISTED @@ -565,7 +519,6 @@ class TestValidator(UploadMixin, TestCase): 'olympia.devhub.tasks.forward_linter_results', 'olympia.scanners.tasks.run_yara', 'olympia.scanners.tasks.run_customs', - 'olympia.scanners.tasks.run_wat', ] assert len(scanners_chord.tasks) == len(expected_parallel_tasks) assert expected_parallel_tasks == [task.name for task in scanners_chord.tasks] diff --git a/src/olympia/devhub/utils.py b/src/olympia/devhub/utils.py index 5fc71f48e9..54de693aec 100644 --- a/src/olympia/devhub/utils.py +++ b/src/olympia/devhub/utils.py @@ -17,7 +17,7 @@ from olympia.amo.urlresolvers import linkify_and_clean from olympia.files.models import File, FileUpload from olympia.files.tasks import repack_fileupload from olympia.files.utils import parse_addon, parse_xpi -from olympia.scanners.tasks import run_customs, run_wat, run_yara, call_mad_api +from olympia.scanners.tasks import run_customs, run_yara, call_mad_api from olympia.versions.models import Version from olympia.versions.utils import process_color_value @@ -265,9 +265,6 @@ class Validator: if waffle.switch_is_active('enable-customs'): tasks_in_parallel.append(run_customs.s(upload_pk)) - if waffle.switch_is_active('enable-wat'): - tasks_in_parallel.append(run_wat.s(upload_pk)) - return [ tasks.create_initial_validation_results.si(), repack_fileupload.s(upload_pk), diff --git a/src/olympia/lib/settings_base.py b/src/olympia/lib/settings_base.py index 6897d5eecd..9036eea085 100644 --- a/src/olympia/lib/settings_base.py +++ b/src/olympia/lib/settings_base.py @@ -856,7 +856,6 @@ CELERY_TASK_ROUTES = { 'olympia.devhub.tasks.validate_upload': {'queue': 'devhub'}, 'olympia.files.tasks.repack_fileupload': {'queue': 'devhub'}, 'olympia.scanners.tasks.run_customs': {'queue': 'devhub'}, - 'olympia.scanners.tasks.run_wat': {'queue': 'devhub'}, 'olympia.scanners.tasks.run_yara': {'queue': 'devhub'}, 'olympia.scanners.tasks.call_mad_api': {'queue': 'devhub'}, # Activity (goes to devhub queue). @@ -1489,8 +1488,6 @@ EXTENSION_WORKSHOP_URL = env( SCANNER_TIMEOUT = 60 # seconds CUSTOMS_API_URL = env('CUSTOMS_API_URL', default=None) CUSTOMS_API_KEY = env('CUSTOMS_API_KEY', default=None) -WAT_API_URL = env('WAT_API_URL', default=None) -WAT_API_KEY = env('WAT_API_KEY', default=None) MAD_API_URL = env('MAD_API_URL', default=None) MAD_API_TIMEOUT = 5 # seconds # Git(Hub) repository names, e.g., `owner/repo-name` diff --git a/src/olympia/scanners/migrations/0046_delete_waffle_switch.py b/src/olympia/scanners/migrations/0046_delete_waffle_switch.py new file mode 100644 index 0000000000..d1f475410d --- /dev/null +++ b/src/olympia/scanners/migrations/0046_delete_waffle_switch.py @@ -0,0 +1,14 @@ +from django.db import migrations + + +def remove_waffle_switch(apps, schema_editor): + Switch = apps.get_model('waffle', 'Switch') + + Switch.objects.filter(name='enable-wat').delete() + + +class Migration(migrations.Migration): + + dependencies = [('scanners', '0045_auto_20210604_1340')] + + operations = [migrations.RunPython(remove_waffle_switch)] diff --git a/src/olympia/scanners/migrations/0047_delete_wat_scanner_results.py b/src/olympia/scanners/migrations/0047_delete_wat_scanner_results.py new file mode 100644 index 0000000000..4413664c4f --- /dev/null +++ b/src/olympia/scanners/migrations/0047_delete_wat_scanner_results.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.13 on 2022-04-25 09:33 + +from django.db import migrations + + +def delete_scanner_results(apps, schema_editor): + ScannerResult = apps.get_model('scanners', 'ScannerResult') + + # 2 = WAT + ScannerResult.objects.filter(scanner=2).delete() + + +class Migration(migrations.Migration): + + dependencies = [('scanners', '0046_delete_waffle_switch')] + + operations = [migrations.RunPython(delete_scanner_results)] diff --git a/src/olympia/scanners/models.py b/src/olympia/scanners/models.py index 6e965919ea..9b5fe57e3d 100644 --- a/src/olympia/scanners/models.py +++ b/src/olympia/scanners/models.py @@ -34,7 +34,6 @@ from olympia.constants.scanners import ( SCANNERS, SCHEDULED, UNKNOWN, - WAT, YARA, ) from olympia.files.models import FileUpload @@ -123,10 +122,10 @@ class AbstractScannerResult(ModelBase): return res def can_report_feedback(self): - return self.state == UNKNOWN and self.scanner not in [WAT, MAD] + return self.state == UNKNOWN and self.scanner not in [MAD] def can_revert_feedback(self): - return self.state != UNKNOWN and self.scanner not in [WAT, MAD] + return self.state != UNKNOWN and self.scanner not in [MAD] def get_git_repository(self): return { diff --git a/src/olympia/scanners/tasks.py b/src/olympia/scanners/tasks.py index b938820986..88d3b2c455 100644 --- a/src/olympia/scanners/tasks.py +++ b/src/olympia/scanners/tasks.py @@ -24,7 +24,6 @@ from olympia.constants.scanners import ( MAD, RUNNING, SCANNERS, - WAT, YARA, ) from olympia.devhub.tasks import validation_task @@ -158,29 +157,6 @@ def run_customs(results, upload_pk): ) -@validation_task -def run_wat(results, upload_pk): - """ - Run the wat scanner on a FileUpload and store the results. - - This task is intended to be run as part of the submission process only. - When a version is created from a FileUpload, the files are removed. In - addition, we usually delete old FileUpload entries after 180 days. - - - `results` are the validation results passed in the validation chain. This - task is a validation task, which is why it must receive the validation - results as first argument. - - `upload_pk` is the FileUpload ID. - """ - return run_scanner( - results, - upload_pk, - scanner=WAT, - api_url=settings.WAT_API_URL, - api_key=settings.WAT_API_KEY, - ) - - @validation_task def run_yara(results, upload_pk): """ diff --git a/src/olympia/scanners/tests/test_admin.py b/src/olympia/scanners/tests/test_admin.py index 56d4e27d38..46435dd70b 100644 --- a/src/olympia/scanners/tests/test_admin.py +++ b/src/olympia/scanners/tests/test_admin.py @@ -34,7 +34,6 @@ from olympia.constants.scanners import ( SCHEDULED, TRUE_POSITIVE, UNKNOWN, - WAT, YARA, ) from olympia.files.models import FileUpload @@ -240,11 +239,6 @@ class TestScannerResultAdmin(TestCase): # version. assert '/browse/' not in self.admin.formatted_matched_rules_with_files(result) - def test_formatted_score_when_scanner_is_not_mad_or_customs(self): - result = ScannerResult(score=0.123, scanner=WAT) - - assert self.admin.formatted_score(result) == '-' - def test_formatted_score_for_customs(self): result = ScannerResult(score=0.123, scanner=CUSTOMS) @@ -264,9 +258,6 @@ class TestScannerResultAdmin(TestCase): ScannerResult.objects.create( scanner=CUSTOMS, version=addon_factory().current_version ) - ScannerResult.objects.create( - scanner=WAT, version=addon_factory().current_version - ) deleted_addon = addon_factory(name='a deleted add-on') ScannerResult.objects.create( scanner=CUSTOMS, version=deleted_addon.current_version diff --git a/src/olympia/scanners/tests/test_models.py b/src/olympia/scanners/tests/test_models.py index da20e442f1..8383b118bc 100644 --- a/src/olympia/scanners/tests/test_models.py +++ b/src/olympia/scanners/tests/test_models.py @@ -21,7 +21,6 @@ from olympia.constants.scanners import ( SCANNERS, SCHEDULED, UNKNOWN, - WAT, YARA, ) from olympia.files.models import FileUpload @@ -47,9 +46,6 @@ class TestScannerResultMixin: def create_customs_result(self): return self.model.objects.create(scanner=CUSTOMS) - def create_wat_result(self): - return self.model.objects.create(scanner=WAT) - def create_mad_result(self): return self.model.objects.create(scanner=MAD) @@ -134,10 +130,6 @@ class TestScannerResultMixin: assert result.extract_rule_names() == [rule1, rule2] - def test_extract_rule_names_returns_empty_list_for_unsupported_scanner(self): - result = self.create_wat_result() - assert result.extract_rule_names() == [] - def test_extract_rule_names_with_no_customs_matched_rules_attribute(self): result = self.create_customs_result() result.results = {} @@ -171,7 +163,7 @@ class TestScannerResultMixin: assert result.get_git_repository() == git_repo def test_get_git_repository_returns_none_if_not_supported(self): - result = self.create_wat_result() + result = self.create_customs_result() assert result.get_git_repository() is None def test_can_report_feedback(self): @@ -183,10 +175,6 @@ class TestScannerResultMixin: result.state = FALSE_POSITIVE assert not result.can_report_feedback() - def test_can_report_feedback_is_false_when_scanner_is_wat(self): - result = self.create_wat_result() - assert not result.can_report_feedback() - def test_can_report_feedback_is_false_when_scanner_is_mad(self): result = self.create_mad_result() assert not result.can_report_feedback() @@ -201,10 +189,6 @@ class TestScannerResultMixin: assert result.state == UNKNOWN assert not result.can_revert_feedback() - def test_get_files_by_matched_rules_for_wat(self): - result = self.create_wat_result() - assert result.get_files_by_matched_rules() == {} - def test_get_files_by_matched_rules_with_no_yara_results(self): result = self.create_yara_result() assert result.get_files_by_matched_rules() == {} @@ -305,10 +289,6 @@ class TestScannerResult(TestScannerResultMixin, TestCase): upload = self.create_file_upload() return self.model.objects.create(upload=upload, scanner=CUSTOMS) - def create_wat_result(self): - upload = self.create_file_upload() - return self.model.objects.create(upload=upload, scanner=WAT) - def create_yara_result(self): upload = self.create_file_upload() return self.model.objects.create(upload=upload, scanner=YARA) @@ -329,10 +309,10 @@ class TestScannerResult(TestScannerResultMixin, TestCase): upload = self.create_file_upload() customs_result = self.model.objects.create(upload=upload, scanner=CUSTOMS) - wat_result = self.model.objects.create(upload=upload, scanner=WAT) + yara_result = self.model.objects.create(upload=upload, scanner=YARA) assert customs_result.scanner == CUSTOMS - assert wat_result.scanner == WAT + assert yara_result.scanner == YARA def test_upload_constraint(self): upload = self.create_file_upload() diff --git a/src/olympia/scanners/tests/test_tasks.py b/src/olympia/scanners/tests/test_tasks.py index d64356b2e3..fc17b73e16 100644 --- a/src/olympia/scanners/tests/test_tasks.py +++ b/src/olympia/scanners/tests/test_tasks.py @@ -24,7 +24,6 @@ from olympia.constants.scanners import ( NEW, RUNNING, SCHEDULED, - WAT, YARA, ) from olympia.files.models import File @@ -41,7 +40,6 @@ from olympia.scanners.tasks import ( mark_yara_query_rule_as_completed_or_aborted, run_customs, run_scanner, - run_wat, run_yara, run_yara_query_rule, run_yara_query_rule_on_versions_chunk, @@ -248,44 +246,6 @@ class TestRunCustoms(TestCase): assert returned_results == self.results -class TestRunWat(TestCase): - API_URL = 'http://wat.example.org' - API_KEY = 'some-api-key' - - def setUp(self): - super().setUp() - - self.upload_pk = 1234 - self.results = {**amo.VALIDATOR_SKELETON_RESULTS} - - @override_settings(WAT_API_URL=API_URL, WAT_API_KEY=API_KEY) - @mock.patch('olympia.scanners.tasks.run_scanner') - def test_calls_run_scanner_with_mock(self, run_scanner_mock): - run_scanner_mock.return_value = self.results - - returned_results = run_wat(self.results, self.upload_pk) - - assert run_scanner_mock.called - run_scanner_mock.assert_called_once_with( - self.results, - self.upload_pk, - scanner=WAT, - api_url=self.API_URL, - api_key=self.API_KEY, - ) - assert returned_results == self.results - - @override_settings(WAT_API_URL=API_URL, WAT_API_KEY=API_KEY) - @mock.patch('olympia.scanners.tasks.run_scanner') - def test_does_not_run_when_results_contain_errors(self, run_scanner_mock): - self.results.update({'errors': 1}) - - returned_results = run_wat(self.results, self.upload_pk) - - assert not run_scanner_mock.called - assert returned_results == self.results - - class TestRunYara(UploadMixin, TestCase): def setUp(self): super().setUp() diff --git a/src/olympia/scanners/tests/test_views.py b/src/olympia/scanners/tests/test_views.py index 14da0d3457..d1b2487d1e 100644 --- a/src/olympia/scanners/tests/test_views.py +++ b/src/olympia/scanners/tests/test_views.py @@ -19,7 +19,6 @@ from olympia.constants.scanners import ( LABEL_BAD, MAD, TRUE_POSITIVE, - WAT, YARA, ) from olympia.blocklist.models import Block @@ -72,7 +71,7 @@ class TestScannerResultViewInternal(TestCase): # result labelled as "good" because it has been approved good_version_1 = version_factory(addon=addon_factory()) good_result_1 = ScannerResult.objects.create( - scanner=WAT, version=good_version_1 + scanner=CUSTOMS, version=good_version_1 ) ActivityLog.create(amo.LOG.APPROVE_VERSION, good_version_1, user=self.user) # result labelled as "good" because auto-approve has been confirmed @@ -158,7 +157,7 @@ class TestScannerResultViewInternal(TestCase): ) # result labelled as "good" because it has been approved good_version = version_factory(addon=addon_factory()) - ScannerResult.objects.create(scanner=WAT, version=good_version) + ScannerResult.objects.create(scanner=CUSTOMS, version=good_version) VersionLog.objects.create( activity_log=ActivityLog.create( action=amo.LOG.APPROVE_VERSION, @@ -197,7 +196,7 @@ class TestScannerResultViewInternal(TestCase): ) # result labelled as "good" because it has been approved good_version = version_factory(addon=addon_factory()) - ScannerResult.objects.create(scanner=WAT, version=good_version) + ScannerResult.objects.create(scanner=CUSTOMS, version=good_version) VersionLog.objects.create( activity_log=ActivityLog.create( action=amo.LOG.APPROVE_VERSION, @@ -215,9 +214,11 @@ class TestScannerResultViewInternal(TestCase): response = self.client.get('{}'.format(f'{self.url}?scanner=yara&label=bad')) self.assert_json_results(response, expected_results=1) - response = self.client.get('{}'.format(f'{self.url}?scanner=wat&label=bad')) + response = self.client.get('{}'.format(f'{self.url}?scanner=customs&label=bad')) self.assert_json_results(response, expected_results=0) - response = self.client.get('{}'.format(f'{self.url}?scanner=wat&label=good')) + response = self.client.get( + '{}'.format(f'{self.url}?scanner=customs&label=good') + ) self.assert_json_results(response, expected_results=1) def test_get_results_with_blocked_versions(self): diff --git a/src/olympia/versions/tests/test_models.py b/src/olympia/versions/tests/test_models.py index ddd3b45f87..6bb836acc6 100644 --- a/src/olympia/versions/tests/test_models.py +++ b/src/olympia/versions/tests/test_models.py @@ -34,7 +34,7 @@ from olympia.constants.promoted import ( SPOTLIGHT, STRATEGIC, ) -from olympia.constants.scanners import CUSTOMS, WAT, YARA, MAD +from olympia.constants.scanners import CUSTOMS, YARA, MAD from olympia.files.models import File from olympia.files.tests.test_models import UploadMixin from olympia.files.utils import parse_addon @@ -1462,21 +1462,6 @@ class TestExtensionVersionFromUpload(TestVersionFromUpload): scanners_result.refresh_from_db() assert scanners_result.version == version - def test_set_version_to_wat_scanners_result(self): - self.create_switch('enable-wat', active=True) - scanners_result = ScannerResult.objects.create(upload=self.upload, scanner=WAT) - assert scanners_result.version is None - - version = Version.from_upload( - self.upload, - self.addon, - amo.RELEASE_CHANNEL_LISTED, - selected_apps=[self.selected_app], - parsed_data=self.dummy_parsed_data, - ) - scanners_result.refresh_from_db() - assert scanners_result.version == version - def test_set_version_to_yara_scanners_result(self): self.create_switch('enable-yara', active=True) scanners_result = ScannerResult.objects.create(upload=self.upload, scanner=YARA) @@ -1495,7 +1480,6 @@ class TestExtensionVersionFromUpload(TestVersionFromUpload): def test_does_nothing_when_no_scanner_is_enabled(self): self.create_switch('enable-customs', active=False) - self.create_switch('enable-wat', active=False) self.create_switch('enable-yara', active=False) scanners_result = ScannerResult.objects.create( upload=self.upload, scanner=CUSTOMS