Add release notes and license info (except full text) to search API

docs/topics/api/addons.rst

@@ -161,7 +161,7 @@ This endpoint allows you to fetch a specific add-on by id, slug or guid.
     :>json array categories[app_name]: Array holding the :ref:`category slugs <category-list>` the add-on belongs to for a given :ref:`add-on application <addon-detail-application>`. (Combine with the add-on ``type`` to determine the name of the category).
     :>json string contributions_url: URL to the (external) webpage where the addon's authors collect monetary contributions, if set. Can be an empty value.
     :>json string created: The date the add-on was created.
-    :>json object current_version: Object holding the current :ref:`version <version-detail-object>` of the add-on. For performance reasons the ``license`` field omits the ``text`` property from the detail endpoint. In addition, ``license`` and ``release_notes`` are omitted entirely from the search endpoint.
+    :>json object current_version: Object holding the current :ref:`version <version-detail-object>` of the add-on. For performance reasons the ``license`` field omits the ``text`` property from both the search and detail endpoints.
     :>json string default_locale: The add-on default locale for translations.
     :>json string|object|null description: The add-on description (See :ref:`translated fields <api-overview-translations>`).
     :>json string|object|null developer comments: Additional information about the add-on provided by the developer. (See :ref:`translated fields <api-overview-translations>`).
@@ -384,12 +384,12 @@ This endpoint allows you to fetch a single version belonging to a specific add-o
     :>json int files[].size: The size for a file, in bytes.
     :>json int files[].status: The :ref:`status <addon-detail-status>` for a file.
     :>json string files[].url: The (absolute) URL to download a file. Clients using this API can append an optional ``src`` query parameter to the url which would indicate the source of the request (See :ref:`download sources <download-sources>`).
-    :>json object license: Object holding information about the license for the version. For performance reasons this field is omitted from add-on search endpoint.
+    :>json object license: Object holding information about the license for the version.
     :>json boolean license.is_custom: Whether the license text has been provided by the developer, or not.  (When ``false`` the license is one of the common, predefined, licenses).
     :>json string|object|null license.name: The name of the license (See :ref:`translated fields <api-overview-translations>`).
     :>json string|object|null license.text: The text of the license (See :ref:`translated fields <api-overview-translations>`). For performance reasons this field is omitted from add-on detail endpoint.
     :>json string|null license.url: The URL of the full text of license.
-    :>json string|object|null release_notes: The release notes for this version (See :ref:`translated fields <api-overview-translations>`). For performance reasons this field is omitted from add-on search endpoint.
+    :>json string|object|null release_notes: The release notes for this version (See :ref:`translated fields <api-overview-translations>`).
     :>json string reviewed: The date the version was reviewed at.
     :>json boolean is_strict_compatibility_enabled: Whether or not this version has `strictCompatibility <https://developer.mozilla.org/en-US/Add-ons/Install_Manifests#strictCompatibility>`_. set.
     :>json string version: The version number string for the version.

docs/topics/api/overview.rst

@@ -291,3 +291,4 @@ v4 API changelog
 * 2018-11-15: added ``is_custom`` to the license object in version detail output in the addons API.
 * 2018-11-22: added ``flags`` to the rating object in the ratings API when ``show_flags_for`` parameter supplied.
 * 2018-11-22: added ``score`` parameter to the ratings API list endpoint.
+* 2019-01-10: added ``release_notes`` and ``license`` (except ``license.text``) to search API results ``current_version`` objects.

src/olympia/addons/indexers.py

@@ -87,6 +87,17 @@ class AddonIndexer(BaseSearchIndexer):
                             'type': 'keyword', 'index': False},
                     }
                 },
+                'license': {
+                    'type': 'object',
+                    'properties': {
+                        'id': {'type': 'long', 'index': False},
+                        'builtin': {'type': 'boolean', 'index': False},
+                        'name_translations': cls.get_translations_definition(),
+                        'url': {'type': 'text', 'index': False}
+                    },
+                },
+                'release_notes_translations':
+                    cls.get_translations_definition(),
                 'version': {'type': 'keyword', 'index': False},
             }
         }
@@ -155,7 +166,7 @@ class AddonIndexer(BaseSearchIndexer):
                         'analyzer': 'standard_with_word_split',
                         'fields': {
                             # Raw field for exact matches and sorting.
-                            'raw': cls.raw_field_definition(),
+                            'raw': cls.get_raw_field_definition(),
                             # Trigrams for partial matches.
                             'trigrams': {
                                 'type': 'text',
@@ -179,6 +190,8 @@ class AddonIndexer(BaseSearchIndexer):
                         'type': 'object',
                         'properties': {
                             'id': {'type': 'long', 'index': False},
+                            'caption_translations':
+                                cls.get_translations_definition(),
                             'modified': {'type': 'date', 'index': False},
                             'sizes': {
                                 'type': 'object',
@@ -228,7 +241,9 @@ class AddonIndexer(BaseSearchIndexer):
 
     @classmethod
     def extract_version(cls, obj, version_obj):
-        return {
+        from olympia.versions.models import License, Version
+
+        data = {
             'id': version_obj.pk,
             'compatible_apps': cls.extract_compatibility_info(
                 obj, version_obj),
@@ -250,6 +265,20 @@ class AddonIndexer(BaseSearchIndexer):
             'reviewed': version_obj.reviewed,
             'version': version_obj.version,
         } if version_obj else None
+        if data and version_obj:
+            attach_trans_dict(Version, [version_obj])
+            data.update(cls.extract_field_api_translations(
+                version_obj, 'release_notes', db_field='releasenotes_id'))
+            if version_obj.license:
+                data['license'] = {
+                    'id': version_obj.license.id,
+                    'builtin': version_obj.license.builtin,
+                    'url': version_obj.license.url,
+                }
+                attach_trans_dict(License, [version_obj.license])
+                data['license'].update(cls.extract_field_api_translations(
+                    version_obj.license, 'name'))
+        return data
 
     @classmethod
     def extract_compatibility_info(cls, obj, version_obj):

src/olympia/addons/serializers.py

@@ -8,7 +8,9 @@ from olympia import amo
 from olympia.accounts.serializers import BaseUserSerializer
 from olympia.amo.templatetags.jinja_helpers import absolutify
 from olympia.amo.urlresolvers import get_outgoing_url, reverse
-from olympia.api.fields import ReverseChoiceField, TranslationSerializerField
+from olympia.api.fields import (
+    ESTranslationSerializerField, ReverseChoiceField,
+    TranslationSerializerField)
 from olympia.api.serializers import BaseESSerializer
 from olympia.api.utils import is_gate_active
 from olympia.applications.models import AppVersion
@@ -193,17 +195,6 @@ class SimpleVersionSerializer(MinimalVersionSerializer):
         return absolutify(obj.get_url_path())
 
 
-class SimpleESVersionSerializer(SimpleVersionSerializer):
-    class Meta:
-        model = Version
-        # In ES, we don't have license and release notes info, so instead of
-        # returning null, which is not necessarily true, we omit those fields
-        # entirely.
-        fields = ('id', 'compatibility', 'edit_url', 'files',
-                  'is_strict_compatibility_enabled', 'reviewed', 'url',
-                  'version')
-
-
 class VersionSerializer(SimpleVersionSerializer):
     channel = ReverseChoiceField(choices=amo.CHANNEL_CHOICES_API.items())
     license = LicenseSerializer()
@@ -253,6 +244,32 @@ class CurrentVersionSerializer(SimpleVersionSerializer):
         return version_qs.first() or addon.current_version
 
 
+class ESCompactLicenseSerializer(BaseESSerializer, CompactLicenseSerializer):
+    translated_fields = ('name', )
+
+    def __init__(self, *args, **kwargs):
+        super(ESCompactLicenseSerializer, self).__init__(*args, **kwargs)
+        self.db_name = ESTranslationSerializerField()
+        self.db_name.bind('name', self)
+
+    def fake_object(self, data):
+        # We just pass the data as the fake object will have been created
+        # before by ESAddonSerializer.fake_version_object()
+        return data
+
+
+class ESCurrentVersionSerializer(BaseESSerializer, CurrentVersionSerializer):
+    license = ESCompactLicenseSerializer()
+
+    datetime_fields = ('reviewed',)
+    translated_fields = ('release_notes',)
+
+    def fake_object(self, data):
+        # We just pass the data as the fake object will have been created
+        # before by ESAddonSerializer.fake_version_object()
+        return data
+
+
 class AddonEulaPolicySerializer(serializers.ModelSerializer):
     eula = TranslationSerializerField()
     privacy_policy = TranslationSerializerField()
@@ -497,7 +514,7 @@ class ESAddonSerializer(BaseESSerializer, AddonSerializer):
     # data the same way than the regular serializer does (usually because we
     # some of the data is not indexed in ES).
     authors = BaseUserSerializer(many=True, source='listed_authors')
-    current_version = SimpleESVersionSerializer()
+    current_version = ESCurrentVersionSerializer()
     previews = ESPreviewSerializer(many=True, source='current_previews')
     _score = serializers.SerializerMethodField()
 
@@ -561,6 +578,23 @@ class ESAddonSerializer(BaseESSerializer, AddonSerializer):
                     min=AppVersion(version=compat_dict.get('min_human', '')),
                     max=AppVersion(version=compat_dict.get('max_human', '')))
             version._compatible_apps = compatible_apps
+            version_serializer = self.fields['current_version']
+            # Can't use version_serializer._attach_translations() directly
+            # because release_notes source field name is different.
+            version_serializer.fields['release_notes'].attach_translations(
+                version, data, 'release_notes', target_name='releasenotes')
+            if 'license' in data:
+                license_serializer = version_serializer.fields['license']
+                version.license = License(id=data['license']['id'])
+                license_serializer._attach_fields(
+                    version.license, data['license'], ('builtin', 'url'))
+                # Can't use license_serializer._attach_translations() directly
+                # because 'name' is a SerializerMethodField, not an
+                # ESTranslatedField.
+                license_serializer.db_name.attach_translations(
+                    version.license, data['license'], 'name')
+            else:
+                version.license = None
         else:
             version = None
         return version

src/olympia/addons/tests/test_indexers.py

@@ -7,14 +7,13 @@ from olympia.addons.models import (
     Addon, Preview, attach_tags, attach_translations)
 from olympia.amo.models import SearchMixin
 from olympia.amo.tests import (
-    ESTestCase, TestCase, addon_factory, collection_factory, file_factory,
-    version_factory)
+    ESTestCase, TestCase, addon_factory, collection_factory, file_factory)
 from olympia.bandwagon.models import FeaturedCollection
 from olympia.constants.applications import FIREFOX
 from olympia.constants.platforms import PLATFORM_ALL, PLATFORM_MAC
 from olympia.constants.search import SEARCH_ANALYZER_MAP
 from olympia.files.models import WebextPermission
-from olympia.versions.models import VersionPreview
+from olympia.versions.models import License, VersionPreview
 
 
 class TestAddonIndexer(TestCase):
@@ -112,7 +111,8 @@ class TestAddonIndexer(TestCase):
         assert mapping_properties['current_version']['properties']
         version_mapping = mapping_properties['current_version']['properties']
         expected_version_keys = (
-            'id', 'compatible_apps', 'files', 'reviewed', 'version')
+            'id', 'compatible_apps', 'files', 'license',
+            'release_notes_translations', 'reviewed', 'version')
         assert set(version_mapping.keys()) == set(expected_version_keys)
 
         # Make sure files mapping is set inside current_version.
@@ -257,17 +257,22 @@ class TestAddonIndexer(TestCase):
 
     def test_extract_version_and_files(self):
         version = self.addon.current_version
-        file_factory(version=version, platform=PLATFORM_MAC.id)
+        # Make the version a webextension and add a bunch of things to it to
+        # test different scenarios.
+        version.all_files[0].update(is_webextension=True)
+        file_factory(
+            version=version, platform=PLATFORM_MAC.id, is_webextension=True)
+        del version.all_files
+        version.license = License.objects.create(
+            name=u'My licensé',
+            url='http://example.com/',
+            builtin=0)
+        [WebextPermission.objects.create(
+            file=file_, permissions=['bookmarks', 'random permission']
+        ) for file_ in version.all_files]
+        version.save()
 
-        unlisted_version = version_factory(
-            addon=self.addon, channel=amo.RELEASE_CHANNEL_UNLISTED, file_kw={
-                'is_webextension': True,
-            })
-        # Give one of the versions some webext permissions to test that.
-        WebextPermission.objects.create(
-            file=unlisted_version.all_files[0],
-            permissions=['bookmarks', 'random permission']
-        )
+        # Now we can run the extraction and start testing.
         extracted = self._extract()
 
         assert extracted['current_version']
@@ -282,6 +287,17 @@ class TestAddonIndexer(TestCase):
                 'min_human': '2.0',
             }
         }
+        assert extracted['current_version']['license'] == {
+            'builtin': 0,
+            'id': version.license.pk,
+            'name_translations': [{'lang': u'en-US', 'string': u'My licensé'}],
+            'url': u'http://example.com/'
+        }
+        assert extracted['current_version']['release_notes_translations'] == [
+            {'lang': 'en-US', 'string': u'Fix for an important bug'},
+            {'lang': 'fr', 'string': u"Quelque chose en fran\xe7ais."
+                                     u"\n\nQuelque chose d'autre."},
+        ]
         assert extracted['current_version']['reviewed'] == version.reviewed
         assert extracted['current_version']['version'] == version.version
         for index, file_ in enumerate(version.all_files):
@@ -298,7 +314,8 @@ class TestAddonIndexer(TestCase):
             assert extracted_file['platform'] == file_.platform
             assert extracted_file['size'] == file_.size
             assert extracted_file['status'] == file_.status
-            assert extracted_file['webext_permissions_list'] == []
+            assert extracted_file['webext_permissions_list'] == [
+                'bookmarks', 'random permission']
 
         assert set(extracted['platforms']) == set([PLATFORM_MAC.id,
                                                    PLATFORM_ALL.id])

src/olympia/addons/tests/test_serializers.py

@@ -877,11 +877,6 @@ class TestESAddonSerializerOutput(AddonSerializerOutputTestMixin, ESTestCase):
             'username': author.username,
         }
 
-    def _test_version_license_and_release_notes(self, version, data):
-        """Override because the ES serializer doesn't include those fields."""
-        assert 'license' not in data
-        assert 'release_notes' not in data
-
     def test_score(self):
         self.request.version = 'v4'
         self.addon = addon_factory()

src/olympia/amo/indexers.py

@@ -52,18 +52,26 @@ class BaseSearchIndexer(object):
 
         for field_name in field_names:
             # _translations is the suffix in TranslationSerializer.
-            mapping[doc_name]['properties'].update({
-                '%s_translations' % field_name: {
-                    'type': 'object',
-                    'properties': {
-                        'lang': {'type': 'text', 'index': False},
-                        'string': {'type': 'text', 'index': False}
-                    }
-                }
-            })
+            mapping[doc_name]['properties']['%s_translations' % field_name] = (
+                cls.get_translations_definition())
 
     @classmethod
-    def raw_field_definition(cls):
+    def get_translations_definition(cls):
+        """
+        Return the mapping to use for raw translations (to be returned directly
+        by the API, not used for analysis).
+        See attach_translation_mappings() for more information.
+        """
+        return {
+            'type': 'object',
+            'properties': {
+                'lang': {'type': 'text', 'index': False},
+                'string': {'type': 'text', 'index': False}
+            }
+        }
+
+    @classmethod
+    def get_raw_field_definition(cls):
         """
         Return the mapping to use for the "raw" version of a field. Meant to be
         used as part of a 'fields': {'raw': ... } definition in the mapping of
@@ -116,7 +124,7 @@ class BaseSearchIndexer(object):
                     'type': 'text',
                     'analyzer': analyzer,
                     'fields': {
-                        'raw': cls.raw_field_definition(),
+                        'raw': cls.get_raw_field_definition(),
                     }
                 }