enhance recreate_theme_previews task to selectively regenerate sizes (#16700)

* enhance recreate_theme_previews task to selectively regenerate sizes

* longer variable names

src/olympia/addons/tasks.py

@@ -25,12 +25,14 @@ from olympia.amo.celery import task
 from olympia.amo.decorators import use_primary_db
 from olympia.amo.storage_utils import copy_stored_file
 from olympia.amo.utils import LocalFileStorage, extract_colors_from_image
+from olympia.devhub.tasks import resize_image
 from olympia.files.models import File
 from olympia.files.utils import get_filepath, parse_addon
 from olympia.lib.es.utils import index_objects
 from olympia.tags.models import Tag
 from olympia.versions.models import (
     generate_static_theme_preview,
+    Version,
     VersionPreview,
 )
 
@@ -191,23 +193,50 @@ def recreate_theme_previews(addon_ids, **kw):
         '[%s@%s] Recreating previews for themes starting at id: %s...'
         % (len(addon_ids), recreate_theme_previews.rate_limit, addon_ids[0])
     )
-    addons = Addon.objects.filter(pk__in=addon_ids).no_transforms()
+    version_ids = Addon.objects.filter(pk__in=addon_ids).values_list('_current_version')
+    versions = Version.objects.filter(pk__in=version_ids)
     only_missing = kw.get('only_missing', False)
 
-    for addon in addons:
-        version = addon.current_version
-        if not version:
-            continue
+    renders = {
+        render['full']: {
+            'thumb_size': render['thumbnail'],
+            'thumb_format': render['thumbnail_format'],
+        }
+        for render in amo.THEME_PREVIEW_RENDERINGS.values()
+    }
+
+    for version in versions:
         try:
             if only_missing:
-                with_size = (
-                    VersionPreview.objects.filter(version=version)
-                    .exclude(sizes={})
-                    .count()
+                existing_full_sizes = {
+                    tuple(size.get('image', ()))
+                    for size in VersionPreview.objects.filter(
+                        version=version
+                    ).values_list('sizes', flat=True)
+                }
+                all_full_sizes_present = not set(renders.keys()) - existing_full_sizes
+                if all_full_sizes_present:
+                    # i.e. we have all renders
+                    for preview in list(VersionPreview.objects.filter(version=version)):
+                        # so check the thumbnail size/format for each preview
+                        render = renders.get(tuple(preview.image_dimensions))
+                        if render and (
+                            render['thumb_size'] != tuple(preview.thumbnail_dimensions)
+                            or render['thumb_format'] != preview.get_format('thumbnail')
+                        ):
+                            preview.sizes['thumbnail_format'] = render['thumb_format']
+                            preview.sizes['thumbnail'] = render['thumb_size']
+                            resize_image(
+                                preview.image_path,
+                                preview.thumbnail_path,
+                                render['thumb_size'],
+                                format=render['thumb_format'],
+                                quality=35,
                             )
-                if with_size == len(amo.THEME_PREVIEW_RENDERINGS):
+                            preview.save()
                     continue
-            log.info('Recreating previews for theme: %s' % addon.id)
+                # else carry on with a full preview generation
+            log.info('Recreating previews for theme: %s' % version.addon_id)
             VersionPreview.objects.filter(version=version).delete()
             xpi = get_filepath(version.all_files[0])
             theme_data = parse_addon(xpi, minimal=True).get('theme', {})

src/olympia/addons/tests/test_tasks.py

@@ -58,38 +58,82 @@ def test_recreate_theme_previews():
     ]
 
 
+PATCH_PATH = 'olympia.addons.tasks'
+
+
 @pytest.mark.django_db
-@mock.patch('olympia.addons.tasks.parse_addon')
+@mock.patch(f'{PATCH_PATH}.parse_addon')
 def test_create_missing_theme_previews(parse_addon_mock):
+
     parse_addon_mock.return_value = {}
     theme = addon_factory(type=amo.ADDON_STATICTHEME)
-    preview = VersionPreview.objects.create(
+    amo_preview = VersionPreview.objects.create(
         version=theme.current_version,
-        sizes={'image': [123, 456], 'thumbnail': [34, 45]},
+        sizes={
+            'image': amo.THEME_PREVIEW_RENDERINGS['amo']['full'],
+            'thumbnail': amo.THEME_PREVIEW_RENDERINGS['amo']['thumbnail'],
+            'thumbnail_format': amo.THEME_PREVIEW_RENDERINGS['amo']['thumbnail_format'],
+        },
     )
-    VersionPreview.objects.create(
+    firefox_preview = VersionPreview.objects.create(
+        version=theme.current_version,
+        sizes={
+            'image': amo.THEME_PREVIEW_RENDERINGS['firefox']['full'],
+            'thumbnail': amo.THEME_PREVIEW_RENDERINGS['firefox']['thumbnail'],
+        },
+    )
+    # add another extra preview size that should be ignored
+    extra_preview = VersionPreview.objects.create(
         version=theme.current_version,
         sizes={'image': [123, 456], 'thumbnail': [34, 45]},
     )
 
-    # addon has 2 complete previews already so skip when only_missing=True
-    with mock.patch('olympia.addons.tasks.generate_static_theme_preview') as p:
+    # addon has all the complete previews already so skip when only_missing=True
+    assert VersionPreview.objects.count() == 3
+    with mock.patch(f'{PATCH_PATH}.generate_static_theme_preview') as gstp, mock.patch(
+        f'{PATCH_PATH}.resize_image'
+    ) as rs:
         recreate_theme_previews([theme.id], only_missing=True)
-        assert p.call_count == 0
+        assert gstp.call_count == 0
+        assert rs.call_count == 0
         recreate_theme_previews([theme.id], only_missing=False)
-        assert p.call_count == 1
+        assert gstp.call_count == 1
+        assert rs.call_count == 0
+        assert VersionPreview.objects.count() == 0
 
-    # break one of the previews
-    preview.update(sizes={})
-    with mock.patch('olympia.addons.tasks.generate_static_theme_preview') as p:
+    # If the add-on is missing a preview, we call generate_static_theme_preview
+    # amo_preview.delete() - `recreate_theme_previews` deletes all the previews already
+    firefox_preview.save()
+    extra_preview.save()
+    assert VersionPreview.objects.count() == 2
+    with mock.patch(f'{PATCH_PATH}.generate_static_theme_preview') as gstp, mock.patch(
+        f'{PATCH_PATH}.resize_image'
+    ) as rs:
         recreate_theme_previews([theme.id], only_missing=True)
-        assert p.call_count == 1
+        assert gstp.call_count == 1
+        assert rs.call_count == 0
 
-    # And delete it so the addon only has 1 preview
-    preview.delete()
-    with mock.patch('olympia.addons.tasks.generate_static_theme_preview') as p:
+    # But we don't do the full regeneration to just get new thumbnail sizes or formats
+    amo_preview.sizes['thumbnail'] = [666, 444]
+    amo_preview.save()
+    assert amo_preview.thumbnail_dimensions == [666, 444]
+    firefox_preview.sizes['thumbnail_format'] = 'gif'
+    firefox_preview.save()
+    assert firefox_preview.get_format('thumbnail') == 'gif'
+    extra_preview.save()
+    assert VersionPreview.objects.count() == 3
+    with mock.patch(f'{PATCH_PATH}.generate_static_theme_preview') as gstp, mock.patch(
+        f'{PATCH_PATH}.resize_image'
+    ) as rs:
         recreate_theme_previews([theme.id], only_missing=True)
-        assert p.call_count == 1
+        assert gstp.call_count == 0  # not called
+        assert rs.call_count == 2
+        amo_preview.reload()
+        assert amo_preview.thumbnail_dimensions == [720, 92]
+        firefox_preview.reload()
+        assert firefox_preview.get_format('thumbnail') == 'png'
+
+        assert VersionPreview.objects.count() == 3
 
 
 @pytest.mark.django_db