From 8c2e3cdd072e671f855aaaae211874102092b400 Mon Sep 17 00:00:00 2001 From: Mathieu Pillard Date: Tue, 7 Jun 2022 15:09:17 +0200 Subject: [PATCH] Re-implement update service in Django (#19322) Re-implement update service in Django Lives in parallel for now - the switch between the 2 will happen through nginx config changes. --- docker/nginx/addons.conf | 2 +- src/olympia/conf/dev/settings.py | 2 - src/olympia/conf/stage/settings.py | 2 - src/olympia/files/utils.py | 10 - src/olympia/lib/settings_base.py | 2 + src/olympia/urls.py | 6 + src/olympia/versions/models.py | 61 +++- src/olympia/versions/tests/test_models.py | 157 +++++++++- src/olympia/versions/tests/test_update.py | 359 ++++++++++++++++++++++ src/olympia/versions/update.py | 126 ++++++++ 10 files changed, 695 insertions(+), 32 deletions(-) create mode 100644 src/olympia/versions/tests/test_update.py create mode 100644 src/olympia/versions/update.py diff --git a/docker/nginx/addons.conf b/docker/nginx/addons.conf index 17746cdac8..8b4adfc438 100644 --- a/docker/nginx/addons.conf +++ b/docker/nginx/addons.conf @@ -5,7 +5,7 @@ merge_slashes off; server { listen 80 default; - location ~ ^/update/.* { + location ~ ^/update-legacy/.* { try_files $uri @update; } diff --git a/src/olympia/conf/dev/settings.py b/src/olympia/conf/dev/settings.py index b17b72a8a1..54693c3ca2 100644 --- a/src/olympia/conf/dev/settings.py +++ b/src/olympia/conf/dev/settings.py @@ -90,8 +90,6 @@ FXA_PROFILE_HOST = 'https://profile.stage.mozaws.net/v1' DEFAULT_FXA_CONFIG_NAME = 'default' ALLOWED_FXA_CONFIGS = ['default', 'local'] -VAMO_URL = 'https://versioncheck-dev.allizom.org' - REMOTE_SETTINGS_IS_TEST_SERVER = True SITEMAP_DEBUG_AVAILABLE = True diff --git a/src/olympia/conf/stage/settings.py b/src/olympia/conf/stage/settings.py index ca2e8ccfa5..01f639e318 100644 --- a/src/olympia/conf/stage/settings.py +++ b/src/olympia/conf/stage/settings.py @@ -88,8 +88,6 @@ TAAR_LITE_RECOMMENDATION_ENGINE_URL = env( default=('https://taarlite.prod.mozaws.net/taarlite/api/v1/addon_recommendations/'), ) -VAMO_URL = 'https://versioncheck.allizom.org' - EXTENSION_WORKSHOP_URL = env( 'EXTENSION_WORKSHOP_URL', default='https://extensionworkshop.allizom.org' ) diff --git a/src/olympia/files/utils.py b/src/olympia/files/utils.py index 1edf5af14f..770060e351 100644 --- a/src/olympia/files/utils.py +++ b/src/olympia/files/utils.py @@ -52,16 +52,6 @@ SIGNED_RE = re.compile(r'^META\-INF/(\w+)\.(rsa|sf)$') # (see toolkit/components/extensions/ExtensionUtils.jsm) MSG_RE = re.compile(r'__MSG_(?P[a-zA-Z0-9@_]+?)__') -# The default update URL. -default = ( - 'https://versioncheck.addons.mozilla.org/update/VersionCheck.php?' - 'reqVersion=%REQ_VERSION%&id=%ITEM_ID%&version=%ITEM_VERSION%&' - 'maxAppVersion=%ITEM_MAXAPPVERSION%&status=%ITEM_STATUS%&appID=%APP_ID%&' - 'appVersion=%APP_VERSION%&appOS=%APP_OS%&appABI=%APP_ABI%&' - 'locale=%APP_LOCALE%¤tAppVersion=%CURRENT_APP_VERSION%&' - 'updateType=%UPDATE_TYPE%' -) - def get_filepath(fileorpath): """Resolve the actual file path of `fileorpath`. diff --git a/src/olympia/lib/settings_base.py b/src/olympia/lib/settings_base.py index dfbbfc9028..f84c7f651f 100644 --- a/src/olympia/lib/settings_base.py +++ b/src/olympia/lib/settings_base.py @@ -272,6 +272,7 @@ SUPPORTED_NONAPPS = ( 'services', 'sitemap.xml', 'static', + 'update', 'user-media', '__heartbeat__', '__lbheartbeat__', @@ -291,6 +292,7 @@ SUPPORTED_NONLOCALES = ( 'sitemap.xml', 'downloads', 'static', + 'update', 'user-media', '__heartbeat__', '__lbheartbeat__', diff --git a/src/olympia/urls.py b/src/olympia/urls.py index 538394d49f..7284060728 100644 --- a/src/olympia/urls.py +++ b/src/olympia/urls.py @@ -9,6 +9,7 @@ from olympia.amo.utils import urlparams from olympia.amo.views import frontend_view from olympia.files.urls import upload_patterns from olympia.versions import views as version_views +from olympia.versions.update import update from olympia.versions.urls import download_patterns @@ -76,6 +77,11 @@ urlpatterns = [ r'^addons/versions/(\d+)/?$', lambda r, id: redirect('addons.versions', id, permanent=True), ), + re_path( + r'^update/(?:VersionCheck\.php)?$', + update, + name='addons.versions.update', + ), # Legacy redirect. Doesn't receive the addon id/slug so it can't live with # the other version views that do. re_path( diff --git a/src/olympia/versions/models.py b/src/olympia/versions/models.py index bfb50ee33d..01d144dc71 100644 --- a/src/olympia/versions/models.py +++ b/src/olympia/versions/models.py @@ -10,7 +10,7 @@ import django.dispatch from django.core.exceptions import ObjectDoesNotExist from django.core.files.storage import default_storage as storage from django.db import models, transaction -from django.db.models import Q +from django.db.models import Case, F, Q, When from django.urls import reverse from django.utils.encoding import force_str from django.utils.functional import cached_property @@ -88,22 +88,54 @@ class VersionManager(ManagerBase): def reviewed(self): return self.filter(file__status__in=amo.REVIEWED_STATUSES) - def latest_public_compatible_with(self, application, appversions): + def latest_public_compatible_with( + self, application, appversions, *, strict_compat_mode=False + ): """Return a queryset filtering the versions so that they are public, listed, and compatible with the application and appversions parameters passed. The queryset is ordered by creation date descending, allowing the caller to get the latest compatible version available. - application is an application id - appversions is a dict containing min and max values, as version ints. + `application` is an application id + `appversions` is a dict containing min and max values, as version ints. + + By default, `appversions['max']` is only considered for versions that + have strict compatibility enabled, unless the `strict_compat_mode` + parameter is also True. + + Regardless of whether appversions are passed or not, the queryset will + be annotated with min_compatible_version and max_compatible_version + values, corresponding to the min and max application version each + Version is compatible with. """ - return Version.objects.filter( - apps__application=application, - apps__min__version_int__lte=appversions['min'], - apps__max__version_int__gte=appversions['max'], - channel=amo.RELEASE_CHANNEL_LISTED, - file__status=amo.STATUS_APPROVED, - ).order_by('-created') + filters = { + 'channel': amo.RELEASE_CHANNEL_LISTED, + 'file__status': amo.STATUS_APPROVED, + } + filters = { + 'channel': amo.RELEASE_CHANNEL_LISTED, + 'file__status': amo.STATUS_APPROVED, + 'apps__application': application, + } + annotations = { + 'min_compatible_version': F('apps__min__version'), + 'max_compatible_version': F('apps__max__version'), + } + if 'min' in appversions: + filters['apps__min__version_int__lte'] = appversions['min'] + + if 'max' in appversions: + if strict_compat_mode: + filters['apps__max__version_int__gte'] = appversions['max'] + else: + filters['apps__max__version_int__gte'] = Case( + When(file__strict_compatibility=True, then=appversions['max']), + default=0, + ) + + # Note: the filter() needs to happen before the annotate(), otherwise + # it would create extra joins! + return self.filter(**filters).annotate(**annotations).order_by('-created') def auto_approvable(self): """Returns a queryset filtered with just the versions that should @@ -249,12 +281,12 @@ class Version(OnChangeMixin, ModelBase): Create a Version instance and corresponding File(s) from a FileUpload, an Addon, a channel id and the parsed_data generated by parse_addon(). Additionally, for non-themes: either a list of compatible app ids - needs to be provided as `selected_apps`, or a list of `ApplicationVersions` + needs to be provided as `selected_apps`, or a list of `ApplicationsVersions` instances for each compatible app as `compatibility`. If `compatibility` is provided: the `version` property of the instances will be set to the new upload and the instances saved. If the min and/or max properties - of the `ApplicationVersions` instance are none then `AppVersion`s parsed from + of the `ApplicationsVersions` instance are none then `AppVersion`s parsed from the manifest, or defaults, are used. Note that it's the caller's responsability to ensure the file is valid. @@ -539,8 +571,7 @@ class Version(OnChangeMixin, ModelBase): def compatible_apps(self): """Returns a mapping of {APP: ApplicationsVersions}. This may have been filled by the transformer already.""" - # Dicts and search providers don't have compatibility info. - # Fake one for them. + # We override whatever compatibility information dictionaries have. if self.addon and self.addon.type in amo.NO_COMPAT: return {app: None for app in amo.APP_TYPE_SUPPORT[self.addon.type]} # Otherwise calculate from the related compat instances. diff --git a/src/olympia/versions/tests/test_models.py b/src/olympia/versions/tests/test_models.py index 53d30b8aa6..ff57153b50 100644 --- a/src/olympia/versions/tests/test_models.py +++ b/src/olympia/versions/tests/test_models.py @@ -65,8 +65,8 @@ from olympia.versions.models import ( pytestmark = pytest.mark.django_db -class TestVersionManager(TestCase): - def test_latest_public_compatible_with(self): +class TestVersionManagerLatestPublicCompatibleWith(TestCase): + def test_latest_public_compatible_with_multiple_addons(self): # Add compatible add-ons. We're going to request versions compatible # with 58.0. compatible_pack1 = addon_factory( @@ -199,6 +199,159 @@ class TestVersionManager(TestCase): ] assert list(qs) == expected_versions + def test_latest_public_compatible_with(self): + addon = addon_factory( + version_kw={'min_app_version': '57.0', 'max_app_version': '57.*'}, + ) + appversions = { + 'min': version_int('58.0'), + 'max': version_int('58.0'), + } + qs = Version.objects.latest_public_compatible_with(amo.FIREFOX.id, appversions) + # We should get 4 joins: + # - applications_versions + # - appversions (min) + # - appversions (max) + # - files (status and strict_compatibility) + assert str(qs.query).count('JOIN') == 4 + # We're not in strict mode, and the add-on hasn't strict compatibility enabled, + # so we find a result. + assert qs.exists() + assert qs[0] == addon.current_version + assert qs[0].min_compatible_version == '57.0' + assert qs[0].max_compatible_version == '57.*' + + def test_latest_public_compatible_with_wrong_app(self): + addon = addon_factory( + version_kw={ + 'application': amo.ANDROID.id, + 'min_app_version': '57.0', + 'max_app_version': '*', + }, + ) + appversions = { + 'min': version_int('58.0'), + 'max': version_int('58.0'), + } + qs = Version.objects.latest_public_compatible_with(amo.FIREFOX.id, appversions) + assert not qs.exists() + assert str(qs.query).count('JOIN') == 4 + + qs = Version.objects.latest_public_compatible_with(amo.ANDROID.id, appversions) + assert qs.exists() + assert str(qs.query).count('JOIN') == 4 + assert qs[0] == addon.current_version + assert qs[0].min_compatible_version == '57.0' + assert qs[0].max_compatible_version == '*' + + # Add a Firefox version, but don't let it be compatible with what we're + # requesting yet. + av_min, _ = AppVersion.objects.get_or_create( + application=amo.FIREFOX.id, version='59.0' + ) + av_max, _ = AppVersion.objects.get_or_create( + application=amo.FIREFOX.id, version='*' + ) + ApplicationsVersions.objects.get_or_create( + application=amo.FIREFOX.id, + version=addon.current_version, + min=av_min, + max=av_max, + ) + qs = Version.objects.latest_public_compatible_with(amo.FIREFOX.id, appversions) + assert not qs.exists() + + av_min.version = '58.0' + av_min.version_int = None + av_min.save() # Will deal with version_intification behind the scenes. + + # Now it should work! + qs = Version.objects.latest_public_compatible_with(amo.FIREFOX.id, appversions) + assert qs.exists() + assert qs[0] == addon.current_version + assert qs[0].min_compatible_version == '58.0' + assert qs[0].max_compatible_version == '*' + + def test_latest_public_compatible_with_no_max_argument(self): + addon = addon_factory( + version_kw={'min_app_version': '57.0', 'max_app_version': '57.*'}, + ) + appversions = { + 'min': version_int('58.0'), + } + qs = Version.objects.latest_public_compatible_with(amo.FIREFOX.id, appversions) + assert str(qs.query).count('JOIN') == 4 + assert qs.exists() + assert qs[0] == addon.current_version + assert qs[0].min_compatible_version == '57.0' + assert qs[0].max_compatible_version == '57.*' # Still annotated. + + def test_latest_public_compatible_with_strict_compat_mode(self): + addon = addon_factory( + version_kw={'min_app_version': '57.0', 'max_app_version': '57.*'}, + ) + appversions = { + 'min': version_int('58.0'), + 'max': version_int('58.0'), + } + qs = Version.objects.latest_public_compatible_with( + amo.FIREFOX.id, appversions, strict_compat_mode=True + ) + assert str(qs.query).count('JOIN') == 4 + assert not qs.exists() + + appversions = { + 'min': version_int('57.0'), + 'max': version_int('57.0'), + } + qs = Version.objects.latest_public_compatible_with( + amo.FIREFOX.id, appversions, strict_compat_mode=True + ) + assert str(qs.query).count('JOIN') == 4 + assert qs.exists() + assert qs[0] == addon.current_version + assert qs[0].min_compatible_version == '57.0' + assert qs[0].max_compatible_version == '57.*' + + def test_latest_public_compatible_with_strict_compatibility_set(self): + addon = addon_factory( + version_kw={'min_app_version': '57.0', 'max_app_version': '57.*'}, + file_kw={'strict_compatibility': True}, + ) + appversions = { + 'min': version_int('58.0'), + 'max': version_int('58.0'), + } + qs = Version.objects.latest_public_compatible_with(amo.FIREFOX.id, appversions) + assert str(qs.query).count('JOIN') == 4 + assert not qs.exists() + + # Strict mode shouldn't change anything. + qs = Version.objects.latest_public_compatible_with( + amo.FIREFOX.id, appversions, strict_compat_mode=True + ) + assert str(qs.query).count('JOIN') == 4 + assert not qs.exists() + + appversions = { + 'min': version_int('57.0'), + 'max': version_int('57.0'), + } + qs = Version.objects.latest_public_compatible_with(amo.FIREFOX.id, appversions) + assert str(qs.query).count('JOIN') == 4 + assert qs.exists() + assert qs[0] == addon.current_version + assert qs[0].min_compatible_version == '57.0' + assert qs[0].max_compatible_version == '57.*' + + # Strict mode shouldn't change anything. + qs2 = Version.objects.latest_public_compatible_with( + amo.FIREFOX.id, appversions, strict_compat_mode=True + ) + assert list(qs2) == list(qs) + + +class TestVersionManager(TestCase): def test_version_hidden_from_related_manager_after_deletion(self): """Test that a version that has been deleted should be hidden from the reverse relations, unless using the specific unfiltered_for_relations diff --git a/src/olympia/versions/tests/test_update.py b/src/olympia/versions/tests/test_update.py new file mode 100644 index 0000000000..bac6936894 --- /dev/null +++ b/src/olympia/versions/tests/test_update.py @@ -0,0 +1,359 @@ +from urllib.parse import quote + +from django.conf import settings +from django.urls import reverse + +import pytest + +from olympia import amo +from olympia.amo.tests import addon_factory, TestCase, version_factory +from olympia.constants.applications import APP_GUIDS +from olympia.versions.models import ApplicationsVersions, AppVersion + + +class TestUpdate(TestCase): + def setUp(self): + self.addon = addon_factory( + version_kw={ + 'min_app_version': '57.0', + 'max_app_version': '*', + 'version': '1.0', + }, + file_kw={'hash': 'fakehash'}, + ) + self.addon.current_version.update(created=self.days_ago(42)) + self.url = reverse('addons.versions.update') + self.query_params = { + 'id': self.addon.guid, + 'appID': amo.FIREFOX.guid, + 'appVersion': '99.0', + } + + def _check_common_headers(self, response): + assert response['Content-Type'] == 'application/json' + assert response['Cache-Control'] == 'max-age=3600' + assert response['ETag'] + assert response['Content-Length'] + + def _check_ok(self, expected_version): + with self.assertNumQueries(2): + # - One query to validate the add-on + # - One query to fetch the update + response = self.client.get(self.url, self.query_params) + assert response.status_code == 200 + self._check_common_headers(response) + data = response.json() + assert len(data['addons'][self.addon.guid]['updates']) + update_data = response.json()['addons'][self.addon.guid]['updates'][0] + assert update_data['version'] == expected_version.version + app = APP_GUIDS[self.query_params['appID']] + assert ( + update_data['applications']['gecko']['strict_min_version'] + == expected_version.compatible_apps[app].min.version + ) + if expected_version.file.strict_compatibility: + assert ( + update_data['applications']['gecko']['strict_max_version'] + == expected_version.compatible_apps[app].max.version + ) + assert update_data['update_hash'] == expected_version.file.hash + assert update_data['update_link'] == ( + f'{settings.SITE_URL}/{app.short}/' + f'downloads/file/{expected_version.file.pk}/' + f'{expected_version.file.pretty_filename}' + ) + if expected_version.release_notes: + assert update_data['update_info_url'] == ( + f'{settings.SITE_URL}/%APP_LOCALE%/{app.short}' + f'/addon/{quote(self.addon.slug)}/versions/' + f'{expected_version.version}/updateinfo/' + ) + + def _check_invalid(self, expected_status_code=400): + if expected_status_code == 400: + # Request is invalid, we shouldn't use the database. + expected_num_queries = 0 + else: + # If we're expected something else than a 400, it means the request + # itself was valid and we had to go to the database to return an + # empty response because the add-on itself was invalid. + expected_num_queries = 1 + with self.assertNumQueries(expected_num_queries): + response = self.client.get(self.url, self.query_params) + assert response.status_code == expected_status_code + self._check_common_headers(response) + assert response.json() == {} + + def _check_no_updates(self): + with self.assertNumQueries(2): + # - One query to validate the add-on + # - One query to fetch the update (which will return nothing) + response = self.client.get(self.url, self.query_params) + assert response.status_code == 200 + self._check_common_headers(response) + assert response.json() == {'addons': {self.addon.guid: {'updates': []}}} + + def test_reverse_no_app_or_locale(self): + assert self.url == '/update/' + + def test_no_app_id(self): + self.query_params.pop('appID') + self._check_invalid() + + def test_no_appversion(self): + self.query_params.pop('appVersion') + self._check_invalid() + + def test_invalid_app_id(self): + self.query_params['appID'] = 'garbag2鎈' + self._check_invalid() + + def test_unknown_addon(self): + self.query_params['id'] = 'unknowné鎈' + self._check_invalid(expected_status_code=200) + + def test_inactive_addon(self): + self.addon.update(disabled_by_user=True) + self._check_invalid(expected_status_code=200) + + def test_deleted_addon(self): + self.addon.delete() + self._check_invalid(expected_status_code=200) + + def test_disabled_addon(self): + self.addon.update(status=amo.STATUS_DISABLED) + self._check_invalid(expected_status_code=200) + + def test_no_versions(self): + self.addon.versions.all().delete() + self._check_no_updates() + + def test_no_updates_because_minimum_appversion_too_low(self): + self.query_params['appVersion'] = '56.0' + self._check_no_updates() + + def test_basic(self): + expected_version = version_factory( + addon=self.addon, + min_app_version='58.0', + max_app_version='*', + version='1.1', + file_kw={'hash': 'fakehash1.1', 'filename': 'webextension_no_id.zip'}, + ) + self._check_ok(expected_version) + + def test_release_notes(self): + self.addon.current_version.release_notes = 'Some release notes' + self.addon.current_version.save() + self._check_ok(self.addon.current_version) + + def test_android_with_release_notes(self): + self.query_params['appID'] = amo.ANDROID.guid + self.addon.current_version.apps.all().update(application=amo.ANDROID.id) + expected_version = version_factory( + addon=self.addon, + application=amo.ANDROID.id, + min_app_version='58.0', + max_app_version='*', + version='1.1', + file_kw={'hash': 'fakehash1.1'}, + release_notes='Some release notes', + ) + self._check_ok(expected_version) + + def test_android_compatible_with_both_android_and_firefox_on_same_version(self): + self.query_params['appID'] = amo.ANDROID.guid + av_min, _ = AppVersion.objects.get_or_create( + application=amo.ANDROID.id, version='57.0' + ) + av_max, _ = AppVersion.objects.get_or_create( + application=amo.ANDROID.id, version='*' + ) + ApplicationsVersions.objects.get_or_create( + application=amo.ANDROID.id, + version=self.addon.current_version, + min=av_min, + max=av_max, + ) + expected_version = version_factory( + addon=self.addon, + application=amo.ANDROID.id, + min_app_version='58.0', + max_app_version='*', + version='1.1', + file_kw={'hash': 'fakehash1.1', 'filename': 'webextension_no_id.zip'}, + release_notes='Some release notes', + ) + self._check_ok(expected_version) + + def test_basic_max_star(self): + expected_version = version_factory( + addon=self.addon, + min_app_version='58.0', + max_app_version='*', + version='1.1', + file_kw={'hash': 'fakehash1.1'}, + ) + self._check_ok(expected_version) + + def test_new_style_guid(self): + self.addon.update(guid='myaddon@') + self.query_params['id'] = 'myaddon@' + self._check_ok(self.addon.current_version) + + def test_min_appversion_low(self): + expected_version = self.addon.current_version + version_factory( + addon=self.addon, + min_app_version='58.0', + max_app_version='*', + version='1.1', + file_kw={'hash': 'fakehash1.1'}, + ) + self.query_params['appVersion'] = '57.0' + # We're requesting a version compatible with 57.0, so the newly added + # version won't do it. We should be served the older version instead. + self._check_ok(expected_version) + + def test_latest_version_is_for_another_app_only(self): + expected_version = self.addon.current_version + version_factory( + addon=self.addon, + min_app_version='58.0', + max_app_version='*', + application=amo.ANDROID.id, + version='1.1', + file_kw={'hash': 'fakehash1.1'}, + ) + self._check_ok(expected_version) + + def test_newer_version_not_compatible_because_of_strict_compatibility(self): + expected_version = self.addon.current_version + version_factory( + addon=self.addon, + min_app_version='58.0', + max_app_version='58.*', + version='1.1', + file_kw={'strict_compatibility': True, 'hash': 'fakehash11'}, + ) + # The newer version has strict compatibility set to 58.* max so it + # won't be picked up as we're on a higher version. + self._check_ok(expected_version) + + def test_newer_version_not_public(self): + expected_version = self.addon.current_version + version_factory( + addon=self.addon, + min_app_version='58.0', + max_app_version='*', + version='1.1', + file_kw={'status': amo.STATUS_AWAITING_REVIEW, 'hash': 'fakehash11'}, + ) + # The newer version is not approved, so it won't be picked up. + self._check_ok(expected_version) + + def test_newer_version_disabled(self): + expected_version = self.addon.current_version + version_factory( + addon=self.addon, + min_app_version='58.0', + max_app_version='*', + version='1.1', + file_kw={'status': amo.STATUS_DISABLED, 'hash': 'fakehash11'}, + ) + # The newer version is disabled, so it won't be picked up. + self._check_ok(expected_version) + + def test_no_unlisted_version(self): + expected_version = self.addon.current_version + version_factory( + addon=self.addon, + min_app_version='58.0', + max_app_version='*', + version='1.1', + channel=amo.RELEASE_CHANNEL_UNLISTED, + file_kw={'hash': 'fakehash1.1'}, + ) + # The newer version is unlisted, so it won't be picked up. + self._check_ok(expected_version) + + def test_no_deleted_version(self): + expected_version = self.addon.current_version + version = version_factory( + addon=self.addon, + min_app_version='58.0', + max_app_version='*', + version='1.1', + file_kw={'hash': 'fakehash1.1'}, + ) + version.delete() + # The newer version is deleted, so it won't be picked up. + self._check_ok(expected_version) + + def test_different_addon(self): + expected_version = self.addon.current_version + addon_factory( + version_kw={ + 'min_app_version': '57.0', + 'max_app_version': '*', + 'version': '1.1', + }, + file_kw={'hash': 'fakehash1.1'}, + ) + # The newer version is for a different add-on so it won't be picked up. + self._check_ok(expected_version) + + def test_no_compat_addon(self): + # No updates will be found for add-ons with no compat in default + # (strict) mode. + self.addon.current_version.apps.all().delete() + self._check_no_updates() + + def test_ignore_mode(self): + self.query_params['compatMode'] = 'ignore' + self.test_basic() + + def test_ignore_mode_strict_compatibility_version(self): + self.query_params['compatMode'] = 'ignore' + expected_version = version_factory( + addon=self.addon, + min_app_version='58.0', + max_app_version='58.*', + version='1.1', + file_kw={'strict_compatibility': True, 'hash': 'fakehash11'}, + ) + # Despite having strict compatibility set to 58.* max, the newer + # version _will_ get picked up as we're in ignore mode. + self._check_ok(expected_version) + + @pytest.mark.xfail(reason='Need to sort out compatibility info for dictionaries') + def test_ignore_mode_no_compat_addon(self): + self.query_params['compatMode'] = 'ignore' + self.addon.current_version.apps.all().delete() + # FIXME: this doesn't work, because the update service is always + # passing the minimum app version, and we just deleted it... + # Investigate what should happen here, because that's not new behavior: + # the old service did that too. So should add-ons marked as having no + # compatibility, like dictionaries, always have one recorded in the db + # anyway ? It seems like they do in production. + # https://github.com/mozilla/addons-server/issues/19323 + self._check_ok(self.addon.current_version) + + def test_normal_mode(self): + self.query_params['compatMode'] = 'normal' + self.test_basic() + + def test_normal_mode_min_appversion_low(self): + self.query_params['compatMode'] = 'normal' + self.test_min_appversion_low() + + def test_normal_mode_strict_compatibility_version(self): + self.query_params['compatMode'] = 'normal' + self.test_newer_version_not_compatible_because_of_strict_compatibility() + + def test_normal_mode_no_compat_addon(self): + # No updates will be found for add-ons with no compat in normal + # mode. + self.query_params['compatMode'] = 'normal' + self.addon.current_version.apps.all().delete() + self._check_no_updates() diff --git a/src/olympia/versions/update.py b/src/olympia/versions/update.py new file mode 100644 index 0000000000..1228f0c723 --- /dev/null +++ b/src/olympia/versions/update.py @@ -0,0 +1,126 @@ +from django.db.models import F +from django.db.transaction import non_atomic_requests +from django.http import JsonResponse +from django.views.decorators.cache import cache_control +from django.urls import reverse + +from olympia.addons.models import Addon +from olympia.amo.templatetags.jinja_helpers import absolutify +from olympia.amo.reverse import override_url_prefix +from olympia.constants import applications +from olympia.versions.compare import version_int +from olympia.versions.models import Version + + +# Valid compatMode parameters +# (see mozilla-central/source/toolkit/mozapps/extensions/internal/XPIInstall.jsm) +COMPAT_MODE_STRICT = 'strict' +COMPAT_MODE_NORMAL = 'normal' +COMPAT_MODE_IGNORE = 'ignore' + + +class Updater: + def __init__(self, data): + self.compat_mode = data.get('compatMode', COMPAT_MODE_STRICT) + self.app = applications.APP_GUIDS.get(data.get('appID')) + self.guid = data.get('id') + self.appversion = data.get('appVersion') + + def check_required_parameters(self): + return self.app and self.guid and self.appversion + + def get_addon_id(self): + return ( + Addon.objects.not_disabled_by_mozilla() + .filter(disabled_by_user=False) + .filter(guid=self.guid) + .values_list('id', flat=True) + ).first() + + def get_update(self, addon_id): + # Compatibility-wise, clients pass appVersion and compatMode query + # parameters. The version we return _always_ need to have a min + # appversion set lower or equal to the appversion passed by the client. + # On top of this: + # - if compat mode is "strict", then the version also needs to have a + # max appversion higher or equal to the appversion passed by the + # client. + # - if compat mode is "normal", then the version also needs to have a + # max appversion higher or equal to the appversion passed by the + # client only if its file has strict compatibility set - otherwise + # it just needs to have a max version set to a value higher than 0. + # - if compat mode is "ignore" or any other value, then all versions + # are considered without looking at their max appversion. + strict_compat_mode = self.compat_mode == COMPAT_MODE_STRICT + client_appversion = version_int(self.appversion) + appversions = {'min': client_appversion} + if strict_compat_mode or self.compat_mode == COMPAT_MODE_NORMAL: + appversions['max'] = client_appversion + return ( + Version.objects.latest_public_compatible_with( + self.app.id, appversions, strict_compat_mode=strict_compat_mode + ) + .select_related('file') + .annotate(addon_slug=F('addon__slug')) # Avoids building an Addon instance. + .no_transforms() + .filter(addon_id=addon_id) + .order_by('-pk') + .first() + ) + + def get_output(self): + if not self.check_required_parameters(): + return self.get_error_output(), 400 + if addon_id := self.get_addon_id(): + if version := self.get_update(addon_id): + contents = self.get_success_output(version) + else: + contents = self.get_no_updates_output() + else: + contents = self.get_error_output() + return contents, 200 + + def get_error_output(self): + return {} + + def get_no_updates_output(self): + return {'addons': {self.guid: {'updates': []}}} + + def get_success_output(self, version): + # The update service bypasses our URL prefixer so we need to override + # the values to send the right thing to the clients. + # For the locale we use the special `%APP_LOCALE%` that Firefox will + # replace with the current locale when using the URL. See + # mozilla-central/source/toolkit/mozapps/extensions/AddonManager.jsm + with override_url_prefix(app_name=self.app.short, locale='%APP_LOCALE%'): + update = { + 'version': version.version, + 'update_link': version.file.get_absolute_url(), + 'applications': { + 'gecko': {'strict_min_version': version.min_compatible_version} + }, + } + if version.file.strict_compatibility: + update['applications']['gecko'][ + 'strict_max_version' + ] = version.max_compatible_version + if version.file.hash: + update['update_hash'] = version.file.hash + if version.release_notes_id: + update['update_info_url'] = absolutify( + reverse( + 'addons.versions.update_info', + # Use our addon_slug annotation instead of version.addon.slug + # to avoid additional queries. + args=(version.addon_slug, version.version), + ) + ) + return {'addons': {self.guid: {'updates': [update]}}} + + +@non_atomic_requests +@cache_control(max_age=60 * 60) +def update(request): + updater = Updater(request.GET) + contents, status_code = updater.get_output() + return JsonResponse(contents, status=status_code)