Remove code related to the wat scanner (#19153)

This commit is contained in:
William Durand 2022-04-26 14:38:35 +02:00 коммит произвёл GitHub
Родитель f5265e7ad2
Коммит 6ea321ace0
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
16 изменённых файлов: 51 добавлений и 203 удалений

Просмотреть файл

@ -4,7 +4,6 @@ services:
worker:
depends_on:
- customs
- wat
customs:
build:
@ -16,14 +15,3 @@ services:
- "nginx:olympia.test"
depends_on:
- web
wat:
build:
context: ./private/addons-wat
environment:
- ALLOWED_ORIGIN=http://olympia.test
- PORT=10102
links:
- "nginx:olympia.test"
depends_on:
- web

Просмотреть файл

@ -7,7 +7,6 @@ This folder should contain some of our private tools. In the following, we descr
Make sure to clone:
1. the `customs` repo in `private/addons-customs-scanner`
2. the `wat` repo in `private/addons-wat`
Specify the [`docker-compose.private.yml`](../docker-compose.private.yml) file to `docker-compose` (together with the default [`docker-compose.yml`](../docker-compose.yml) file) to build the Docker images:
@ -30,15 +29,6 @@ $ make shell
$ [root@<docker>:/code#] ./manage.py waffle_switch enable-customs on
```
### wat
A waffle switch is used to enable/disable the `wat` Celery task:
```
$ make shell
$ [root@<docker>:/code#] ./manage.py waffle_switch enable-wat on
```
## Yara
If you have access to `yara`, you should first clone it in `private/addons-yara`.

Просмотреть файл

@ -152,8 +152,6 @@ INBOUND_EMAIL_VALIDATION_KEY = 'totally-unsecure-validation-string'
# Sectools
CUSTOMS_API_URL = 'http://customs:10101/'
CUSTOMS_API_KEY = 'customssecret'
WAT_API_URL = 'http://wat:10102/'
WAT_API_KEY = 'watsecret'
REMOTE_SETTINGS_IS_TEST_SERVER = True

Просмотреть файл

@ -2,11 +2,14 @@ from django.utils.translation import gettext_lazy as _
CUSTOMS = 1
WAT = 2
# We do not use the WAT scanner anymore but we keep this constant for the model
# definition. We shouldn't use this constant, though.
# See: https://github.com/mozilla/addons-server/issues/19152
_WAT = 2
YARA = 3
MAD = 4
SCANNERS = {CUSTOMS: 'customs', WAT: 'wat', YARA: 'yara', MAD: 'mad'}
SCANNERS = {CUSTOMS: 'customs', _WAT: 'wat', YARA: 'yara', MAD: 'mad'}
# Action IDs are also used for severity (the higher, the more severe).
# The field is a PositiveSmallIntegerField, it should go up to 65535.

Просмотреть файл

@ -27,7 +27,7 @@ from olympia.applications.models import AppVersion
from olympia.devhub import tasks, utils
from olympia.files.tasks import repack_fileupload
from olympia.files.tests.test_models import UploadMixin
from olympia.scanners.tasks import run_customs, run_wat, run_yara, call_mad_api
from olympia.scanners.tasks import run_customs, run_yara, call_mad_api
from olympia.versions.models import Version
@ -439,49 +439,6 @@ class TestValidator(UploadMixin, TestCase):
tasks.handle_upload_validation_result.s(file_upload.pk, channel, False),
)
@mock.patch('olympia.devhub.utils.chain')
def test_adds_run_wat_when_enabled(self, mock_chain):
self.create_switch('enable-wat', active=True)
file_upload = self.get_upload('webextension.xpi', with_validation=False)
channel = amo.RELEASE_CHANNEL_LISTED
utils.Validator(file_upload, listed=True)
mock_chain.assert_called_once_with(
tasks.create_initial_validation_results.si(),
repack_fileupload.s(file_upload.pk),
tasks.validate_upload.s(file_upload.pk, channel),
tasks.check_for_api_keys_in_file.s(file_upload.pk),
chord(
[
tasks.forward_linter_results.s(file_upload.pk),
run_wat.s(file_upload.pk),
],
call_mad_api.s(file_upload.pk),
),
tasks.handle_upload_validation_result.s(file_upload.pk, channel, False),
)
@mock.patch('olympia.devhub.utils.chain')
def test_does_not_add_run_wat_when_disabled(self, mock_chain):
self.create_switch('enable-wat', active=False)
file_upload = self.get_upload('webextension.xpi', with_validation=False)
channel = amo.RELEASE_CHANNEL_LISTED
utils.Validator(file_upload, listed=True)
mock_chain.assert_called_once_with(
tasks.create_initial_validation_results.si(),
repack_fileupload.s(file_upload.pk),
tasks.validate_upload.s(file_upload.pk, channel),
tasks.check_for_api_keys_in_file.s(file_upload.pk),
chord(
[tasks.forward_linter_results.s(file_upload.pk)],
call_mad_api.s(file_upload.pk),
),
tasks.handle_upload_validation_result.s(file_upload.pk, channel, False),
)
@mock.patch('olympia.devhub.utils.chain')
def test_adds_yara_and_customs(self, mock_chain):
self.create_switch('enable-customs', active=True)
@ -510,7 +467,6 @@ class TestValidator(UploadMixin, TestCase):
@mock.patch('olympia.devhub.utils.chain')
def test_adds_all_scanners(self, mock_chain):
self.create_switch('enable-customs', active=True)
self.create_switch('enable-wat', active=True)
self.create_switch('enable-yara', active=True)
file_upload = self.get_upload('webextension.xpi', with_validation=False)
channel = amo.RELEASE_CHANNEL_LISTED
@ -527,7 +483,6 @@ class TestValidator(UploadMixin, TestCase):
tasks.forward_linter_results.s(file_upload.pk),
run_yara.s(file_upload.pk),
run_customs.s(file_upload.pk),
run_wat.s(file_upload.pk),
],
call_mad_api.s(file_upload.pk),
),
@ -536,7 +491,6 @@ class TestValidator(UploadMixin, TestCase):
def test_create_file_upload_tasks(self):
self.create_switch('enable-customs', active=True)
self.create_switch('enable-wat', active=True)
self.create_switch('enable-yara', active=True)
file_upload = self.get_upload('webextension.xpi', with_validation=False)
channel = amo.RELEASE_CHANNEL_LISTED
@ -565,7 +519,6 @@ class TestValidator(UploadMixin, TestCase):
'olympia.devhub.tasks.forward_linter_results',
'olympia.scanners.tasks.run_yara',
'olympia.scanners.tasks.run_customs',
'olympia.scanners.tasks.run_wat',
]
assert len(scanners_chord.tasks) == len(expected_parallel_tasks)
assert expected_parallel_tasks == [task.name for task in scanners_chord.tasks]

Просмотреть файл

@ -17,7 +17,7 @@ from olympia.amo.urlresolvers import linkify_and_clean
from olympia.files.models import File, FileUpload
from olympia.files.tasks import repack_fileupload
from olympia.files.utils import parse_addon, parse_xpi
from olympia.scanners.tasks import run_customs, run_wat, run_yara, call_mad_api
from olympia.scanners.tasks import run_customs, run_yara, call_mad_api
from olympia.versions.models import Version
from olympia.versions.utils import process_color_value
@ -265,9 +265,6 @@ class Validator:
if waffle.switch_is_active('enable-customs'):
tasks_in_parallel.append(run_customs.s(upload_pk))
if waffle.switch_is_active('enable-wat'):
tasks_in_parallel.append(run_wat.s(upload_pk))
return [
tasks.create_initial_validation_results.si(),
repack_fileupload.s(upload_pk),

Просмотреть файл

@ -856,7 +856,6 @@ CELERY_TASK_ROUTES = {
'olympia.devhub.tasks.validate_upload': {'queue': 'devhub'},
'olympia.files.tasks.repack_fileupload': {'queue': 'devhub'},
'olympia.scanners.tasks.run_customs': {'queue': 'devhub'},
'olympia.scanners.tasks.run_wat': {'queue': 'devhub'},
'olympia.scanners.tasks.run_yara': {'queue': 'devhub'},
'olympia.scanners.tasks.call_mad_api': {'queue': 'devhub'},
# Activity (goes to devhub queue).
@ -1489,8 +1488,6 @@ EXTENSION_WORKSHOP_URL = env(
SCANNER_TIMEOUT = 60 # seconds
CUSTOMS_API_URL = env('CUSTOMS_API_URL', default=None)
CUSTOMS_API_KEY = env('CUSTOMS_API_KEY', default=None)
WAT_API_URL = env('WAT_API_URL', default=None)
WAT_API_KEY = env('WAT_API_KEY', default=None)
MAD_API_URL = env('MAD_API_URL', default=None)
MAD_API_TIMEOUT = 5 # seconds
# Git(Hub) repository names, e.g., `owner/repo-name`

Просмотреть файл

@ -0,0 +1,14 @@
from django.db import migrations
def remove_waffle_switch(apps, schema_editor):
Switch = apps.get_model('waffle', 'Switch')
Switch.objects.filter(name='enable-wat').delete()
class Migration(migrations.Migration):
dependencies = [('scanners', '0045_auto_20210604_1340')]
operations = [migrations.RunPython(remove_waffle_switch)]

Просмотреть файл

@ -0,0 +1,17 @@
# Generated by Django 3.2.13 on 2022-04-25 09:33
from django.db import migrations
def delete_scanner_results(apps, schema_editor):
ScannerResult = apps.get_model('scanners', 'ScannerResult')
# 2 = WAT
ScannerResult.objects.filter(scanner=2).delete()
class Migration(migrations.Migration):
dependencies = [('scanners', '0046_delete_waffle_switch')]
operations = [migrations.RunPython(delete_scanner_results)]

Просмотреть файл

@ -34,7 +34,6 @@ from olympia.constants.scanners import (
SCANNERS,
SCHEDULED,
UNKNOWN,
WAT,
YARA,
)
from olympia.files.models import FileUpload
@ -123,10 +122,10 @@ class AbstractScannerResult(ModelBase):
return res
def can_report_feedback(self):
return self.state == UNKNOWN and self.scanner not in [WAT, MAD]
return self.state == UNKNOWN and self.scanner not in [MAD]
def can_revert_feedback(self):
return self.state != UNKNOWN and self.scanner not in [WAT, MAD]
return self.state != UNKNOWN and self.scanner not in [MAD]
def get_git_repository(self):
return {

Просмотреть файл

@ -24,7 +24,6 @@ from olympia.constants.scanners import (
MAD,
RUNNING,
SCANNERS,
WAT,
YARA,
)
from olympia.devhub.tasks import validation_task
@ -158,29 +157,6 @@ def run_customs(results, upload_pk):
)
@validation_task
def run_wat(results, upload_pk):
"""
Run the wat scanner on a FileUpload and store the results.
This task is intended to be run as part of the submission process only.
When a version is created from a FileUpload, the files are removed. In
addition, we usually delete old FileUpload entries after 180 days.
- `results` are the validation results passed in the validation chain. This
task is a validation task, which is why it must receive the validation
results as first argument.
- `upload_pk` is the FileUpload ID.
"""
return run_scanner(
results,
upload_pk,
scanner=WAT,
api_url=settings.WAT_API_URL,
api_key=settings.WAT_API_KEY,
)
@validation_task
def run_yara(results, upload_pk):
"""

Просмотреть файл

@ -34,7 +34,6 @@ from olympia.constants.scanners import (
SCHEDULED,
TRUE_POSITIVE,
UNKNOWN,
WAT,
YARA,
)
from olympia.files.models import FileUpload
@ -240,11 +239,6 @@ class TestScannerResultAdmin(TestCase):
# version.
assert '/browse/' not in self.admin.formatted_matched_rules_with_files(result)
def test_formatted_score_when_scanner_is_not_mad_or_customs(self):
result = ScannerResult(score=0.123, scanner=WAT)
assert self.admin.formatted_score(result) == '-'
def test_formatted_score_for_customs(self):
result = ScannerResult(score=0.123, scanner=CUSTOMS)
@ -264,9 +258,6 @@ class TestScannerResultAdmin(TestCase):
ScannerResult.objects.create(
scanner=CUSTOMS, version=addon_factory().current_version
)
ScannerResult.objects.create(
scanner=WAT, version=addon_factory().current_version
)
deleted_addon = addon_factory(name='a deleted add-on')
ScannerResult.objects.create(
scanner=CUSTOMS, version=deleted_addon.current_version

Просмотреть файл

@ -21,7 +21,6 @@ from olympia.constants.scanners import (
SCANNERS,
SCHEDULED,
UNKNOWN,
WAT,
YARA,
)
from olympia.files.models import FileUpload
@ -47,9 +46,6 @@ class TestScannerResultMixin:
def create_customs_result(self):
return self.model.objects.create(scanner=CUSTOMS)
def create_wat_result(self):
return self.model.objects.create(scanner=WAT)
def create_mad_result(self):
return self.model.objects.create(scanner=MAD)
@ -134,10 +130,6 @@ class TestScannerResultMixin:
assert result.extract_rule_names() == [rule1, rule2]
def test_extract_rule_names_returns_empty_list_for_unsupported_scanner(self):
result = self.create_wat_result()
assert result.extract_rule_names() == []
def test_extract_rule_names_with_no_customs_matched_rules_attribute(self):
result = self.create_customs_result()
result.results = {}
@ -171,7 +163,7 @@ class TestScannerResultMixin:
assert result.get_git_repository() == git_repo
def test_get_git_repository_returns_none_if_not_supported(self):
result = self.create_wat_result()
result = self.create_customs_result()
assert result.get_git_repository() is None
def test_can_report_feedback(self):
@ -183,10 +175,6 @@ class TestScannerResultMixin:
result.state = FALSE_POSITIVE
assert not result.can_report_feedback()
def test_can_report_feedback_is_false_when_scanner_is_wat(self):
result = self.create_wat_result()
assert not result.can_report_feedback()
def test_can_report_feedback_is_false_when_scanner_is_mad(self):
result = self.create_mad_result()
assert not result.can_report_feedback()
@ -201,10 +189,6 @@ class TestScannerResultMixin:
assert result.state == UNKNOWN
assert not result.can_revert_feedback()
def test_get_files_by_matched_rules_for_wat(self):
result = self.create_wat_result()
assert result.get_files_by_matched_rules() == {}
def test_get_files_by_matched_rules_with_no_yara_results(self):
result = self.create_yara_result()
assert result.get_files_by_matched_rules() == {}
@ -305,10 +289,6 @@ class TestScannerResult(TestScannerResultMixin, TestCase):
upload = self.create_file_upload()
return self.model.objects.create(upload=upload, scanner=CUSTOMS)
def create_wat_result(self):
upload = self.create_file_upload()
return self.model.objects.create(upload=upload, scanner=WAT)
def create_yara_result(self):
upload = self.create_file_upload()
return self.model.objects.create(upload=upload, scanner=YARA)
@ -329,10 +309,10 @@ class TestScannerResult(TestScannerResultMixin, TestCase):
upload = self.create_file_upload()
customs_result = self.model.objects.create(upload=upload, scanner=CUSTOMS)
wat_result = self.model.objects.create(upload=upload, scanner=WAT)
yara_result = self.model.objects.create(upload=upload, scanner=YARA)
assert customs_result.scanner == CUSTOMS
assert wat_result.scanner == WAT
assert yara_result.scanner == YARA
def test_upload_constraint(self):
upload = self.create_file_upload()

Просмотреть файл

@ -24,7 +24,6 @@ from olympia.constants.scanners import (
NEW,
RUNNING,
SCHEDULED,
WAT,
YARA,
)
from olympia.files.models import File
@ -41,7 +40,6 @@ from olympia.scanners.tasks import (
mark_yara_query_rule_as_completed_or_aborted,
run_customs,
run_scanner,
run_wat,
run_yara,
run_yara_query_rule,
run_yara_query_rule_on_versions_chunk,
@ -248,44 +246,6 @@ class TestRunCustoms(TestCase):
assert returned_results == self.results
class TestRunWat(TestCase):
API_URL = 'http://wat.example.org'
API_KEY = 'some-api-key'
def setUp(self):
super().setUp()
self.upload_pk = 1234
self.results = {**amo.VALIDATOR_SKELETON_RESULTS}
@override_settings(WAT_API_URL=API_URL, WAT_API_KEY=API_KEY)
@mock.patch('olympia.scanners.tasks.run_scanner')
def test_calls_run_scanner_with_mock(self, run_scanner_mock):
run_scanner_mock.return_value = self.results
returned_results = run_wat(self.results, self.upload_pk)
assert run_scanner_mock.called
run_scanner_mock.assert_called_once_with(
self.results,
self.upload_pk,
scanner=WAT,
api_url=self.API_URL,
api_key=self.API_KEY,
)
assert returned_results == self.results
@override_settings(WAT_API_URL=API_URL, WAT_API_KEY=API_KEY)
@mock.patch('olympia.scanners.tasks.run_scanner')
def test_does_not_run_when_results_contain_errors(self, run_scanner_mock):
self.results.update({'errors': 1})
returned_results = run_wat(self.results, self.upload_pk)
assert not run_scanner_mock.called
assert returned_results == self.results
class TestRunYara(UploadMixin, TestCase):
def setUp(self):
super().setUp()

Просмотреть файл

@ -19,7 +19,6 @@ from olympia.constants.scanners import (
LABEL_BAD,
MAD,
TRUE_POSITIVE,
WAT,
YARA,
)
from olympia.blocklist.models import Block
@ -72,7 +71,7 @@ class TestScannerResultViewInternal(TestCase):
# result labelled as "good" because it has been approved
good_version_1 = version_factory(addon=addon_factory())
good_result_1 = ScannerResult.objects.create(
scanner=WAT, version=good_version_1
scanner=CUSTOMS, version=good_version_1
)
ActivityLog.create(amo.LOG.APPROVE_VERSION, good_version_1, user=self.user)
# result labelled as "good" because auto-approve has been confirmed
@ -158,7 +157,7 @@ class TestScannerResultViewInternal(TestCase):
)
# result labelled as "good" because it has been approved
good_version = version_factory(addon=addon_factory())
ScannerResult.objects.create(scanner=WAT, version=good_version)
ScannerResult.objects.create(scanner=CUSTOMS, version=good_version)
VersionLog.objects.create(
activity_log=ActivityLog.create(
action=amo.LOG.APPROVE_VERSION,
@ -197,7 +196,7 @@ class TestScannerResultViewInternal(TestCase):
)
# result labelled as "good" because it has been approved
good_version = version_factory(addon=addon_factory())
ScannerResult.objects.create(scanner=WAT, version=good_version)
ScannerResult.objects.create(scanner=CUSTOMS, version=good_version)
VersionLog.objects.create(
activity_log=ActivityLog.create(
action=amo.LOG.APPROVE_VERSION,
@ -215,9 +214,11 @@ class TestScannerResultViewInternal(TestCase):
response = self.client.get('{}'.format(f'{self.url}?scanner=yara&label=bad'))
self.assert_json_results(response, expected_results=1)
response = self.client.get('{}'.format(f'{self.url}?scanner=wat&label=bad'))
response = self.client.get('{}'.format(f'{self.url}?scanner=customs&label=bad'))
self.assert_json_results(response, expected_results=0)
response = self.client.get('{}'.format(f'{self.url}?scanner=wat&label=good'))
response = self.client.get(
'{}'.format(f'{self.url}?scanner=customs&label=good')
)
self.assert_json_results(response, expected_results=1)
def test_get_results_with_blocked_versions(self):

Просмотреть файл

@ -34,7 +34,7 @@ from olympia.constants.promoted import (
SPOTLIGHT,
STRATEGIC,
)
from olympia.constants.scanners import CUSTOMS, WAT, YARA, MAD
from olympia.constants.scanners import CUSTOMS, YARA, MAD
from olympia.files.models import File
from olympia.files.tests.test_models import UploadMixin
from olympia.files.utils import parse_addon
@ -1462,21 +1462,6 @@ class TestExtensionVersionFromUpload(TestVersionFromUpload):
scanners_result.refresh_from_db()
assert scanners_result.version == version
def test_set_version_to_wat_scanners_result(self):
self.create_switch('enable-wat', active=True)
scanners_result = ScannerResult.objects.create(upload=self.upload, scanner=WAT)
assert scanners_result.version is None
version = Version.from_upload(
self.upload,
self.addon,
amo.RELEASE_CHANNEL_LISTED,
selected_apps=[self.selected_app],
parsed_data=self.dummy_parsed_data,
)
scanners_result.refresh_from_db()
assert scanners_result.version == version
def test_set_version_to_yara_scanners_result(self):
self.create_switch('enable-yara', active=True)
scanners_result = ScannerResult.objects.create(upload=self.upload, scanner=YARA)
@ -1495,7 +1480,6 @@ class TestExtensionVersionFromUpload(TestVersionFromUpload):
def test_does_nothing_when_no_scanner_is_enabled(self):
self.create_switch('enable-customs', active=False)
self.create_switch('enable-wat', active=False)
self.create_switch('enable-yara', active=False)
scanners_result = ScannerResult.objects.create(
upload=self.upload, scanner=CUSTOMS