Remove code related to the wat scanner (#19153)

docker-compose.private.yml

@@ -4,7 +4,6 @@ services:
   worker:
     depends_on:
       - customs
-      - wat
 
   customs:
     build:
@@ -16,14 +15,3 @@ services:
       - "nginx:olympia.test"
     depends_on:
       - web
-
-  wat:
-    build:
-      context: ./private/addons-wat
-    environment:
-      - ALLOWED_ORIGIN=http://olympia.test
-      - PORT=10102
-    links:
-      - "nginx:olympia.test"
-    depends_on:
-      - web

private/README.md

@@ -7,7 +7,6 @@ This folder should contain some of our private tools. In the following, we descr
 Make sure to clone:
 
 1. the `customs` repo in `private/addons-customs-scanner`
-2. the `wat` repo in `private/addons-wat`
 
 Specify the [`docker-compose.private.yml`](../docker-compose.private.yml) file to `docker-compose` (together with the default [`docker-compose.yml`](../docker-compose.yml) file) to build the Docker images:
 
@@ -30,15 +29,6 @@ $ make shell
 $ [root@<docker>:/code#] ./manage.py waffle_switch enable-customs on
 ```
 
-### wat
-
-A waffle switch is used to enable/disable the `wat` Celery task:
-
-```
-$ make shell
-$ [root@<docker>:/code#] ./manage.py waffle_switch enable-wat on
-```
-
 ## Yara
 
 If you have access to `yara`, you should first clone it in `private/addons-yara`.

settings.py

@@ -152,8 +152,6 @@ INBOUND_EMAIL_VALIDATION_KEY = 'totally-unsecure-validation-string'
 # Sectools
 CUSTOMS_API_URL = 'http://customs:10101/'
 CUSTOMS_API_KEY = 'customssecret'
-WAT_API_URL = 'http://wat:10102/'
-WAT_API_KEY = 'watsecret'
 
 REMOTE_SETTINGS_IS_TEST_SERVER = True
 

src/olympia/constants/scanners.py

@@ -2,11 +2,14 @@ from django.utils.translation import gettext_lazy as _
 
 
 CUSTOMS = 1
-WAT = 2
+# We do not use the WAT scanner anymore but we keep this constant for the model
+# definition. We shouldn't use this constant, though.
+# See: https://github.com/mozilla/addons-server/issues/19152
+_WAT = 2
 YARA = 3
 MAD = 4
 
-SCANNERS = {CUSTOMS: 'customs', WAT: 'wat', YARA: 'yara', MAD: 'mad'}
+SCANNERS = {CUSTOMS: 'customs', _WAT: 'wat', YARA: 'yara', MAD: 'mad'}
 
 # Action IDs are also used for severity (the higher, the more severe).
 # The field is a PositiveSmallIntegerField, it should go up to 65535.

src/olympia/devhub/tests/test_utils.py

@@ -27,7 +27,7 @@ from olympia.applications.models import AppVersion
 from olympia.devhub import tasks, utils
 from olympia.files.tasks import repack_fileupload
 from olympia.files.tests.test_models import UploadMixin
-from olympia.scanners.tasks import run_customs, run_wat, run_yara, call_mad_api
+from olympia.scanners.tasks import run_customs, run_yara, call_mad_api
 from olympia.versions.models import Version
 
 
@@ -439,49 +439,6 @@ class TestValidator(UploadMixin, TestCase):
             tasks.handle_upload_validation_result.s(file_upload.pk, channel, False),
         )
 
-    @mock.patch('olympia.devhub.utils.chain')
-    def test_adds_run_wat_when_enabled(self, mock_chain):
-        self.create_switch('enable-wat', active=True)
-        file_upload = self.get_upload('webextension.xpi', with_validation=False)
-        channel = amo.RELEASE_CHANNEL_LISTED
-
-        utils.Validator(file_upload, listed=True)
-
-        mock_chain.assert_called_once_with(
-            tasks.create_initial_validation_results.si(),
-            repack_fileupload.s(file_upload.pk),
-            tasks.validate_upload.s(file_upload.pk, channel),
-            tasks.check_for_api_keys_in_file.s(file_upload.pk),
-            chord(
-                [
-                    tasks.forward_linter_results.s(file_upload.pk),
-                    run_wat.s(file_upload.pk),
-                ],
-                call_mad_api.s(file_upload.pk),
-            ),
-            tasks.handle_upload_validation_result.s(file_upload.pk, channel, False),
-        )
-
-    @mock.patch('olympia.devhub.utils.chain')
-    def test_does_not_add_run_wat_when_disabled(self, mock_chain):
-        self.create_switch('enable-wat', active=False)
-        file_upload = self.get_upload('webextension.xpi', with_validation=False)
-        channel = amo.RELEASE_CHANNEL_LISTED
-
-        utils.Validator(file_upload, listed=True)
-
-        mock_chain.assert_called_once_with(
-            tasks.create_initial_validation_results.si(),
-            repack_fileupload.s(file_upload.pk),
-            tasks.validate_upload.s(file_upload.pk, channel),
-            tasks.check_for_api_keys_in_file.s(file_upload.pk),
-            chord(
-                [tasks.forward_linter_results.s(file_upload.pk)],
-                call_mad_api.s(file_upload.pk),
-            ),
-            tasks.handle_upload_validation_result.s(file_upload.pk, channel, False),
-        )
-
     @mock.patch('olympia.devhub.utils.chain')
     def test_adds_yara_and_customs(self, mock_chain):
         self.create_switch('enable-customs', active=True)
@@ -510,7 +467,6 @@ class TestValidator(UploadMixin, TestCase):
     @mock.patch('olympia.devhub.utils.chain')
     def test_adds_all_scanners(self, mock_chain):
         self.create_switch('enable-customs', active=True)
-        self.create_switch('enable-wat', active=True)
         self.create_switch('enable-yara', active=True)
         file_upload = self.get_upload('webextension.xpi', with_validation=False)
         channel = amo.RELEASE_CHANNEL_LISTED
@@ -527,7 +483,6 @@ class TestValidator(UploadMixin, TestCase):
                     tasks.forward_linter_results.s(file_upload.pk),
                     run_yara.s(file_upload.pk),
                     run_customs.s(file_upload.pk),
-                    run_wat.s(file_upload.pk),
                 ],
                 call_mad_api.s(file_upload.pk),
             ),
@@ -536,7 +491,6 @@ class TestValidator(UploadMixin, TestCase):
 
     def test_create_file_upload_tasks(self):
         self.create_switch('enable-customs', active=True)
-        self.create_switch('enable-wat', active=True)
         self.create_switch('enable-yara', active=True)
         file_upload = self.get_upload('webextension.xpi', with_validation=False)
         channel = amo.RELEASE_CHANNEL_LISTED
@@ -565,7 +519,6 @@ class TestValidator(UploadMixin, TestCase):
             'olympia.devhub.tasks.forward_linter_results',
             'olympia.scanners.tasks.run_yara',
             'olympia.scanners.tasks.run_customs',
-            'olympia.scanners.tasks.run_wat',
         ]
         assert len(scanners_chord.tasks) == len(expected_parallel_tasks)
         assert expected_parallel_tasks == [task.name for task in scanners_chord.tasks]

src/olympia/devhub/utils.py

@@ -17,7 +17,7 @@ from olympia.amo.urlresolvers import linkify_and_clean
 from olympia.files.models import File, FileUpload
 from olympia.files.tasks import repack_fileupload
 from olympia.files.utils import parse_addon, parse_xpi
-from olympia.scanners.tasks import run_customs, run_wat, run_yara, call_mad_api
+from olympia.scanners.tasks import run_customs, run_yara, call_mad_api
 from olympia.versions.models import Version
 from olympia.versions.utils import process_color_value
 
@@ -265,9 +265,6 @@ class Validator:
         if waffle.switch_is_active('enable-customs'):
             tasks_in_parallel.append(run_customs.s(upload_pk))
 
-        if waffle.switch_is_active('enable-wat'):
-            tasks_in_parallel.append(run_wat.s(upload_pk))
-
         return [
             tasks.create_initial_validation_results.si(),
             repack_fileupload.s(upload_pk),

src/olympia/lib/settings_base.py

@@ -856,7 +856,6 @@ CELERY_TASK_ROUTES = {
     'olympia.devhub.tasks.validate_upload': {'queue': 'devhub'},
     'olympia.files.tasks.repack_fileupload': {'queue': 'devhub'},
     'olympia.scanners.tasks.run_customs': {'queue': 'devhub'},
-    'olympia.scanners.tasks.run_wat': {'queue': 'devhub'},
     'olympia.scanners.tasks.run_yara': {'queue': 'devhub'},
     'olympia.scanners.tasks.call_mad_api': {'queue': 'devhub'},
     # Activity (goes to devhub queue).
@@ -1489,8 +1488,6 @@ EXTENSION_WORKSHOP_URL = env(
 SCANNER_TIMEOUT = 60  # seconds
 CUSTOMS_API_URL = env('CUSTOMS_API_URL', default=None)
 CUSTOMS_API_KEY = env('CUSTOMS_API_KEY', default=None)
-WAT_API_URL = env('WAT_API_URL', default=None)
-WAT_API_KEY = env('WAT_API_KEY', default=None)
 MAD_API_URL = env('MAD_API_URL', default=None)
 MAD_API_TIMEOUT = 5  # seconds
 # Git(Hub) repository names, e.g., `owner/repo-name`

src/olympia/scanners/migrations/0046_delete_waffle_switch.py

@@ -0,0 +1,14 @@
+from django.db import migrations
+
+
+def remove_waffle_switch(apps, schema_editor):
+    Switch = apps.get_model('waffle', 'Switch')
+
+    Switch.objects.filter(name='enable-wat').delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [('scanners', '0045_auto_20210604_1340')]
+
+    operations = [migrations.RunPython(remove_waffle_switch)]

src/olympia/scanners/migrations/0047_delete_wat_scanner_results.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.13 on 2022-04-25 09:33
+
+from django.db import migrations
+
+
+def delete_scanner_results(apps, schema_editor):
+    ScannerResult = apps.get_model('scanners', 'ScannerResult')
+
+    # 2 = WAT
+    ScannerResult.objects.filter(scanner=2).delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [('scanners', '0046_delete_waffle_switch')]
+
+    operations = [migrations.RunPython(delete_scanner_results)]

src/olympia/scanners/models.py

@@ -34,7 +34,6 @@ from olympia.constants.scanners import (
     SCANNERS,
     SCHEDULED,
     UNKNOWN,
-    WAT,
     YARA,
 )
 from olympia.files.models import FileUpload
@@ -123,10 +122,10 @@ class AbstractScannerResult(ModelBase):
         return res
 
     def can_report_feedback(self):
-        return self.state == UNKNOWN and self.scanner not in [WAT, MAD]
+        return self.state == UNKNOWN and self.scanner not in [MAD]
 
     def can_revert_feedback(self):
-        return self.state != UNKNOWN and self.scanner not in [WAT, MAD]
+        return self.state != UNKNOWN and self.scanner not in [MAD]
 
     def get_git_repository(self):
         return {

src/olympia/scanners/tasks.py

@@ -24,7 +24,6 @@ from olympia.constants.scanners import (
     MAD,
     RUNNING,
     SCANNERS,
-    WAT,
     YARA,
 )
 from olympia.devhub.tasks import validation_task
@@ -158,29 +157,6 @@ def run_customs(results, upload_pk):
     )
 
 
-@validation_task
-def run_wat(results, upload_pk):
-    """
-    Run the wat scanner on a FileUpload and store the results.
-
-    This task is intended to be run as part of the submission process only.
-    When a version is created from a FileUpload, the files are removed. In
-    addition, we usually delete old FileUpload entries after 180 days.
-
-    - `results` are the validation results passed in the validation chain. This
-       task is a validation task, which is why it must receive the validation
-       results as first argument.
-    - `upload_pk` is the FileUpload ID.
-    """
-    return run_scanner(
-        results,
-        upload_pk,
-        scanner=WAT,
-        api_url=settings.WAT_API_URL,
-        api_key=settings.WAT_API_KEY,
-    )
-
-
 @validation_task
 def run_yara(results, upload_pk):
     """

src/olympia/scanners/tests/test_admin.py

@@ -34,7 +34,6 @@ from olympia.constants.scanners import (
     SCHEDULED,
     TRUE_POSITIVE,
     UNKNOWN,
-    WAT,
     YARA,
 )
 from olympia.files.models import FileUpload
@@ -240,11 +239,6 @@ class TestScannerResultAdmin(TestCase):
         # version.
         assert '/browse/' not in self.admin.formatted_matched_rules_with_files(result)
 
-    def test_formatted_score_when_scanner_is_not_mad_or_customs(self):
-        result = ScannerResult(score=0.123, scanner=WAT)
-
-        assert self.admin.formatted_score(result) == '-'
-
     def test_formatted_score_for_customs(self):
         result = ScannerResult(score=0.123, scanner=CUSTOMS)
 
@@ -264,9 +258,6 @@ class TestScannerResultAdmin(TestCase):
         ScannerResult.objects.create(
             scanner=CUSTOMS, version=addon_factory().current_version
         )
-        ScannerResult.objects.create(
-            scanner=WAT, version=addon_factory().current_version
-        )
         deleted_addon = addon_factory(name='a deleted add-on')
         ScannerResult.objects.create(
             scanner=CUSTOMS, version=deleted_addon.current_version

src/olympia/scanners/tests/test_models.py

@@ -21,7 +21,6 @@ from olympia.constants.scanners import (
     SCANNERS,
     SCHEDULED,
     UNKNOWN,
-    WAT,
     YARA,
 )
 from olympia.files.models import FileUpload
@@ -47,9 +46,6 @@ class TestScannerResultMixin:
     def create_customs_result(self):
         return self.model.objects.create(scanner=CUSTOMS)
 
-    def create_wat_result(self):
-        return self.model.objects.create(scanner=WAT)
-
     def create_mad_result(self):
         return self.model.objects.create(scanner=MAD)
 
@@ -134,10 +130,6 @@ class TestScannerResultMixin:
 
         assert result.extract_rule_names() == [rule1, rule2]
 
-    def test_extract_rule_names_returns_empty_list_for_unsupported_scanner(self):
-        result = self.create_wat_result()
-        assert result.extract_rule_names() == []
-
     def test_extract_rule_names_with_no_customs_matched_rules_attribute(self):
         result = self.create_customs_result()
         result.results = {}
@@ -171,7 +163,7 @@ class TestScannerResultMixin:
             assert result.get_git_repository() == git_repo
 
     def test_get_git_repository_returns_none_if_not_supported(self):
-        result = self.create_wat_result()
+        result = self.create_customs_result()
         assert result.get_git_repository() is None
 
     def test_can_report_feedback(self):
@@ -183,10 +175,6 @@ class TestScannerResultMixin:
         result.state = FALSE_POSITIVE
         assert not result.can_report_feedback()
 
-    def test_can_report_feedback_is_false_when_scanner_is_wat(self):
-        result = self.create_wat_result()
-        assert not result.can_report_feedback()
-
     def test_can_report_feedback_is_false_when_scanner_is_mad(self):
         result = self.create_mad_result()
         assert not result.can_report_feedback()
@@ -201,10 +189,6 @@ class TestScannerResultMixin:
         assert result.state == UNKNOWN
         assert not result.can_revert_feedback()
 
-    def test_get_files_by_matched_rules_for_wat(self):
-        result = self.create_wat_result()
-        assert result.get_files_by_matched_rules() == {}
-
     def test_get_files_by_matched_rules_with_no_yara_results(self):
         result = self.create_yara_result()
         assert result.get_files_by_matched_rules() == {}
@@ -305,10 +289,6 @@ class TestScannerResult(TestScannerResultMixin, TestCase):
         upload = self.create_file_upload()
         return self.model.objects.create(upload=upload, scanner=CUSTOMS)
 
-    def create_wat_result(self):
-        upload = self.create_file_upload()
-        return self.model.objects.create(upload=upload, scanner=WAT)
-
     def create_yara_result(self):
         upload = self.create_file_upload()
         return self.model.objects.create(upload=upload, scanner=YARA)
@@ -329,10 +309,10 @@ class TestScannerResult(TestScannerResultMixin, TestCase):
         upload = self.create_file_upload()
 
         customs_result = self.model.objects.create(upload=upload, scanner=CUSTOMS)
-        wat_result = self.model.objects.create(upload=upload, scanner=WAT)
+        yara_result = self.model.objects.create(upload=upload, scanner=YARA)
 
         assert customs_result.scanner == CUSTOMS
-        assert wat_result.scanner == WAT
+        assert yara_result.scanner == YARA
 
     def test_upload_constraint(self):
         upload = self.create_file_upload()

src/olympia/scanners/tests/test_tasks.py

@@ -24,7 +24,6 @@ from olympia.constants.scanners import (
     NEW,
     RUNNING,
     SCHEDULED,
-    WAT,
     YARA,
 )
 from olympia.files.models import File
@@ -41,7 +40,6 @@ from olympia.scanners.tasks import (
     mark_yara_query_rule_as_completed_or_aborted,
     run_customs,
     run_scanner,
-    run_wat,
     run_yara,
     run_yara_query_rule,
     run_yara_query_rule_on_versions_chunk,
@@ -248,44 +246,6 @@ class TestRunCustoms(TestCase):
         assert returned_results == self.results
 
 
-class TestRunWat(TestCase):
-    API_URL = 'http://wat.example.org'
-    API_KEY = 'some-api-key'
-
-    def setUp(self):
-        super().setUp()
-
-        self.upload_pk = 1234
-        self.results = {**amo.VALIDATOR_SKELETON_RESULTS}
-
-    @override_settings(WAT_API_URL=API_URL, WAT_API_KEY=API_KEY)
-    @mock.patch('olympia.scanners.tasks.run_scanner')
-    def test_calls_run_scanner_with_mock(self, run_scanner_mock):
-        run_scanner_mock.return_value = self.results
-
-        returned_results = run_wat(self.results, self.upload_pk)
-
-        assert run_scanner_mock.called
-        run_scanner_mock.assert_called_once_with(
-            self.results,
-            self.upload_pk,
-            scanner=WAT,
-            api_url=self.API_URL,
-            api_key=self.API_KEY,
-        )
-        assert returned_results == self.results
-
-    @override_settings(WAT_API_URL=API_URL, WAT_API_KEY=API_KEY)
-    @mock.patch('olympia.scanners.tasks.run_scanner')
-    def test_does_not_run_when_results_contain_errors(self, run_scanner_mock):
-        self.results.update({'errors': 1})
-
-        returned_results = run_wat(self.results, self.upload_pk)
-
-        assert not run_scanner_mock.called
-        assert returned_results == self.results
-
-
 class TestRunYara(UploadMixin, TestCase):
     def setUp(self):
         super().setUp()

src/olympia/scanners/tests/test_views.py

@@ -19,7 +19,6 @@ from olympia.constants.scanners import (
     LABEL_BAD,
     MAD,
     TRUE_POSITIVE,
-    WAT,
     YARA,
 )
 from olympia.blocklist.models import Block
@@ -72,7 +71,7 @@ class TestScannerResultViewInternal(TestCase):
         # result labelled as "good" because it has been approved
         good_version_1 = version_factory(addon=addon_factory())
         good_result_1 = ScannerResult.objects.create(
-            scanner=WAT, version=good_version_1
+            scanner=CUSTOMS, version=good_version_1
         )
         ActivityLog.create(amo.LOG.APPROVE_VERSION, good_version_1, user=self.user)
         # result labelled as "good" because auto-approve has been confirmed
@@ -158,7 +157,7 @@ class TestScannerResultViewInternal(TestCase):
         )
         # result labelled as "good" because it has been approved
         good_version = version_factory(addon=addon_factory())
-        ScannerResult.objects.create(scanner=WAT, version=good_version)
+        ScannerResult.objects.create(scanner=CUSTOMS, version=good_version)
         VersionLog.objects.create(
             activity_log=ActivityLog.create(
                 action=amo.LOG.APPROVE_VERSION,
@@ -197,7 +196,7 @@ class TestScannerResultViewInternal(TestCase):
         )
         # result labelled as "good" because it has been approved
         good_version = version_factory(addon=addon_factory())
-        ScannerResult.objects.create(scanner=WAT, version=good_version)
+        ScannerResult.objects.create(scanner=CUSTOMS, version=good_version)
         VersionLog.objects.create(
             activity_log=ActivityLog.create(
                 action=amo.LOG.APPROVE_VERSION,
@@ -215,9 +214,11 @@ class TestScannerResultViewInternal(TestCase):
         response = self.client.get('{}'.format(f'{self.url}?scanner=yara&label=bad'))
         self.assert_json_results(response, expected_results=1)
 
-        response = self.client.get('{}'.format(f'{self.url}?scanner=wat&label=bad'))
+        response = self.client.get('{}'.format(f'{self.url}?scanner=customs&label=bad'))
         self.assert_json_results(response, expected_results=0)
-        response = self.client.get('{}'.format(f'{self.url}?scanner=wat&label=good'))
+        response = self.client.get(
+            '{}'.format(f'{self.url}?scanner=customs&label=good')
+        )
         self.assert_json_results(response, expected_results=1)
 
     def test_get_results_with_blocked_versions(self):

src/olympia/versions/tests/test_models.py

@@ -34,7 +34,7 @@ from olympia.constants.promoted import (
     SPOTLIGHT,
     STRATEGIC,
 )
-from olympia.constants.scanners import CUSTOMS, WAT, YARA, MAD
+from olympia.constants.scanners import CUSTOMS, YARA, MAD
 from olympia.files.models import File
 from olympia.files.tests.test_models import UploadMixin
 from olympia.files.utils import parse_addon
@@ -1462,21 +1462,6 @@ class TestExtensionVersionFromUpload(TestVersionFromUpload):
         scanners_result.refresh_from_db()
         assert scanners_result.version == version
 
-    def test_set_version_to_wat_scanners_result(self):
-        self.create_switch('enable-wat', active=True)
-        scanners_result = ScannerResult.objects.create(upload=self.upload, scanner=WAT)
-        assert scanners_result.version is None
-
-        version = Version.from_upload(
-            self.upload,
-            self.addon,
-            amo.RELEASE_CHANNEL_LISTED,
-            selected_apps=[self.selected_app],
-            parsed_data=self.dummy_parsed_data,
-        )
-        scanners_result.refresh_from_db()
-        assert scanners_result.version == version
-
     def test_set_version_to_yara_scanners_result(self):
         self.create_switch('enable-yara', active=True)
         scanners_result = ScannerResult.objects.create(upload=self.upload, scanner=YARA)
@@ -1495,7 +1480,6 @@ class TestExtensionVersionFromUpload(TestVersionFromUpload):
 
     def test_does_nothing_when_no_scanner_is_enabled(self):
         self.create_switch('enable-customs', active=False)
-        self.create_switch('enable-wat', active=False)
         self.create_switch('enable-yara', active=False)
         scanners_result = ScannerResult.objects.create(
             upload=self.upload, scanner=CUSTOMS