Include disabled add-on when generating stats (#8976)

* Include disabled add-on when generating stats

* Simplify

* Fix filter, add test

* More test updates

* Add test for update_counts_from_file too


Fix #8975

src/olympia/stats/fixtures/files/src/download_counts.hive

@@ -6,3 +6,5 @@
 2014-07-10	1	 666	search
 2014-07-10	1	 7661	search
 2014-07-10	1	 a3615	search
+2014-07-10	1	 disabled-addon	search
+2014-07-10	1	 incomplete-addon	search

src/olympia/stats/fixtures/files/src/update_counts_by_status.hive

@@ -1,3 +1,5 @@
 2014-07-10	{2fa4ed95-0317-4c6a-a74c-5f3e3912c1f9}	userEnabled	2	112
 2014-07-10	{2fa4ed95-0317-4c6a-a74c-5f3e3912c1f9}	userEnabled	2	112
 2014-07-10	{2fa4ed95-0317-4c6a-a74c-5f3e3912c1f9}	userDisabled	1	112
+2014-07-10	{39e6cf40-02f6-4bda-b1ee-409910ffd9f9}	userEnabled	2	112
+2014-07-10	{9c444b87-1124-4fd2-b97f-8fb7e9be1820}	userEnabled	2	112

src/olympia/stats/management/commands/download_counts_from_file.py

@@ -111,6 +111,7 @@ class Command(BaseCommand):
 
         # Memoize the files to addon relations and the DownloadCounts.
         download_counts = {}
+
         # Perf: preload all the files and slugs once and for all.
         # This builds two dicts:
         # - One where each key (the file_id we get from the hive query) has
@@ -118,7 +119,9 @@ class Command(BaseCommand):
         # - One where each key (the add-on slug) has the add-on_id as value.
         files_to_addon = dict(File.objects.values_list('id',
                                                        'version__addon_id'))
-        slugs_to_addon = dict(Addon.objects.public().values_list('slug', 'id'))
+        slugs_to_addon = dict(
+            Addon.unfiltered.exclude(status=amo.STATUS_NULL)
+            .values_list('slug', 'id'))
 
         # Only accept valid sources, which are constants. The source must
         # either be exactly one of the "full" valid sources, or prefixed by one

src/olympia/stats/management/commands/update_counts_from_file.py

@@ -133,10 +133,12 @@ class Command(BaseCommand):
         # Perf: preload all the addons once and for all.
         # This builds a dict where each key (the addon guid we get from the
         # hive query) has the addon_id as value.
-        guids_to_addon = (dict(Addon.objects.public()
-                                            .exclude(guid__isnull=True)
-                                            .exclude(type=amo.ADDON_PERSONA)
-                                            .values_list('guid', 'id')))
+        guids_to_addon = (dict(
+            Addon.unfiltered
+            .exclude(status=amo.STATUS_NULL)
+            .exclude(guid__isnull=True)
+            .exclude(type=amo.ADDON_PERSONA)
+            .values_list('guid', 'id')))
 
         for group, filepath in group_filepaths:
             count_file = get_stats_data(filepath)

src/olympia/stats/tests/test_commands.py

@@ -77,6 +77,36 @@ class TestADICommand(FixturesFolderMixin, TransactionTestCase):
         assert update_count.oses == {u'WINNT': 5}
         assert update_count.locales == {u'en-us': 1, u'en-US': 4}
 
+    def test_update_counts_from_file_includes_disabled_addons(self):
+        addon_factory(
+            guid='{39e6cf40-02f6-4bda-b1ee-409910ffd9f9}',
+            slug='disabled-addon',
+            status=amo.STATUS_DISABLED)
+        addon_factory(
+            guid='9c444b87-1124-4fd2-b97f-8fb7e9be1820',
+            slug='incomplete-addon', status=amo.STATUS_NULL)
+
+        management.call_command('update_counts_from_file', hive_folder,
+                                date=self.date, stats_source=self.stats_source)
+        assert UpdateCount.objects.all().count() == 2
+
+        update_count = UpdateCount.objects.get(addon_id=3615)
+        # should be identical to `statuses.userEnabled`
+        assert update_count.count == 4
+        assert update_count.date == date(2014, 7, 10)
+        assert update_count.versions == {u'3.8': 2, u'3.7': 3}
+        assert update_count.statuses == {u'userDisabled': 1, u'userEnabled': 4}
+
+        update_count = UpdateCount.objects.get(addon__slug='disabled-addon')
+        assert update_count.count == 2
+        assert update_count.date == date(2014, 7, 10)
+        assert update_count.versions == {}
+        assert update_count.statuses == {u'userEnabled': 2}
+
+        # Make sure we didn't generate any stats for incomplete add-ons
+        assert not UpdateCount.objects.filter(
+            addon__slug='incomplete-addon').exists()
+
     def test_update_version(self):
         # Initialize the known addons and their versions.
         self.command.addons_versions = {3615: ['3.5', '3.6']}
@@ -183,6 +213,30 @@ class TestADICommand(FixturesFolderMixin, TransactionTestCase):
         assert download_count.date == date(2014, 7, 10)
         assert download_count.sources == {u'search': 2, u'cb-dl-bob': 1}
 
+    def test_download_counts_from_file_includes_disabled_addons(self):
+        # We only exclude STATUS_NULL add-ons
+        addon_factory(slug='disabled-addon', status=amo.STATUS_DISABLED)
+        addon_factory(slug='incomplete-addon', status=amo.STATUS_NULL)
+
+        management.call_command('download_counts_from_file', hive_folder,
+                                date=self.date, stats_source=self.stats_source)
+
+        assert DownloadCount.objects.all().count() == 3
+        download_count = DownloadCount.objects.get(addon_id=3615)
+        assert download_count.count == 3
+        assert download_count.date == date(2014, 7, 10)
+        assert download_count.sources == {u'search': 2, u'cb-dl-bob': 1}
+
+        download_count = DownloadCount.objects.get(
+            addon__slug='disabled-addon')
+        assert download_count.count == 1
+        assert download_count.date == date(2014, 7, 10)
+        assert download_count.sources == {u'search': 1}
+
+        # Make sure we didn't generate any stats for incomplete add-ons
+        assert not DownloadCount.objects.filter(
+            addon__slug='incomplete-addon').exists()
+
     @mock.patch(
         'olympia.stats.management.commands.download_counts_from_file.'
         'close_old_connections')