Bug 1730918 - use default security groups as defined by product in Bugzilla (#7281)

Support for filing issues as security bugs got added in bug 1369067. Having the
security group hardcoded and not matching the default one for the component
causes bugs to get missed during triage.

docker/entrypoint_prod.sh

@@ -66,6 +66,9 @@ elif [ "$1" == "update_bugscache" ]; then
 elif [ "$1" == "update_files_bugzilla_map" ]; then
     newrelic-admin run-program ./manage.py update_files_bugzilla_map
 
+elif [ "$1" == "update_bugzilla_security_groups" ]; then
+    newrelic-admin run-program ./manage.py update_bugzilla_security_groups
+
 elif [ "$1" == "cache_failure_history" ]; then
     newrelic-admin run-program ./manage.py cache_failure_history
 

newrelic.ini

@@ -30,4 +30,4 @@ shutdown_timeout = 15
 # List finite-duration commands here to enable their annotation by the agent.
 # For infinite duration commands (such as `pulse_listener_*`) see:
 # https://docs.newrelic.com/docs/agents/python-agent/supported-features/python-background-tasks#wrapping
-instrumentation.scripts.django_admin = update_changelog check cycle_data load_initial_data perf_sheriff report_backfill_outcome migrate update_bugscache update_files_bugzilla_map run_intermittents_commenter synthesize_backfill_report backfill_text_log_error_jobs
+instrumentation.scripts.django_admin = update_changelog check cycle_data load_initial_data perf_sheriff report_backfill_outcome migrate update_bugscache update_files_bugzilla_map update_bugzilla_security_groups run_intermittents_commenter synthesize_backfill_report backfill_text_log_error_jobs

tests/webapp/api/test_bugzilla.py

@@ -50,7 +50,7 @@ def test_create_bug(client, eleven_jobs_stored, activate_responses, test_user):
             "comment": u"Intermittent Description",
             "comment_tags": "treeherder",
             "keywords": ["intermittent-failure"],
-            "groups": [],
+            "is_security_issue": False,
         },
     )
     assert resp.status_code == 200
@@ -103,7 +103,7 @@ def test_create_bug_with_unicode(client, eleven_jobs_stored, activate_responses,
             "comment": u"Intermittent “description” string",
             "comment_tags": "treeherder",
             "keywords": ["intermittent-failure"],
-            "groups": [],
+            "is_security_issue": False,
         },
     )
     assert resp.status_code == 200
@@ -158,7 +158,7 @@ def test_create_crash_bug(client, eleven_jobs_stored, activate_responses, test_u
             "crash_signature": "[@crashsig]",
             "priority": "--",
             "keywords": ["intermittent-failure", "crash"],
-            "groups": [],
+            "is_security_issue": False,
         },
     )
     assert resp.status_code == 200
@@ -207,7 +207,7 @@ def test_create_unauthenticated_bug(client, eleven_jobs_stored, activate_respons
             "comment_tags": "treeherder",
             "keywords": ["intermittent-failure"],
             "see_also": "12345",
-            "groups": [],
+            "is_security_issue": False,
         },
     )
     assert resp.status_code == 403
@@ -265,7 +265,7 @@ def test_create_bug_with_long_crash_signature(
             "crash_signature": crashsig,
             "regressed_by": "123",
             "see_also": "12345",
-            "groups": [],
+            "is_security_issue": False,
         },
     )
     assert resp.status_code == 400

treeherder/etl/files_bugzilla_map.py

@@ -1,6 +1,12 @@
 import logging
+import sys
 
-from treeherder.model.models import BugzillaComponent, FilesBugzillaMap, Repository
+from treeherder.model.models import (
+    BugzillaComponent,
+    BugzillaSecurityGroup,
+    FilesBugzillaMap,
+    Repository,
+)
 from treeherder.utils.github import fetch_json
 
 logger = logging.getLogger(__name__)
@@ -25,7 +31,7 @@ class FilesBugzillaMapProcess:
             component = product_component_data[1]
             if len(product) > self.max_product_length:
                 logger.error(
-                    "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): product is too long (has %d characters, max is %d",
+                    "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): product is too long (has %d characters, max is %d)",
                     product,
                     component,
                     path,
@@ -35,17 +41,17 @@ class FilesBugzillaMapProcess:
                 return
             if len(component) > self.max_component_length:
                 logger.error(
-                    "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): component is too long (has %d characters, max is %d",
+                    "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): component is too long (has %d characters, max is %d)",
                     product,
                     component,
                     path,
-                    len(product),
+                    len(component),
                     self.max_component_length,
                 )
                 return
             if len(path) > self.max_path_length:
                 logger.error(
-                    "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): path is too long (has %d characters, max is %d",
+                    "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): path is too long (has %d characters, max is %d)",
                     product,
                     component,
                     path,
@@ -189,3 +195,73 @@ class FilesBugzillaMapProcess:
         )
         bugzilla_components_unused = bugzilla_components_all.difference(bugzilla_components_used)
         (BugzillaComponent.objects.filter(id__in=bugzilla_components_unused).delete())
+
+
+class ProductSecurityGroupProcess:
+    max_product_length = BugzillaSecurityGroup._meta.get_field('product').max_length
+    max_security_group_length = BugzillaSecurityGroup._meta.get_field('security_group').max_length
+
+    def fetch_data(self):
+        url = 'https://bugzilla.mozilla.org/latest/configuration'
+        product_security_group_data = None
+        exception = None
+        try:
+            product_security_group_data = fetch_json(url)
+        except Exception as e:
+            exception = e
+        return {
+            "url": url,
+            "product_security_group_data": product_security_group_data,
+            "exception": exception,
+        }
+
+    def run(self):
+        data_returned = self.fetch_data()
+        if data_returned["exception"] is not None:
+            logger.error(
+                "error fetching file with map of source paths to Bugzilla products and components: url: %s ; %s",
+                data_returned["url"],
+                data_returned["exception"],
+            )
+            sys.exit()
+        fields_data = data_returned["product_security_group_data"]["field"]["product"]["values"]
+        groups_data = data_returned["product_security_group_data"]["group"]
+        products = set()
+        for field_data in fields_data:
+            product_name = str(field_data["name"])
+            security_group_id = str(field_data["security_group_id"])
+            if security_group_id in groups_data:
+                security_group_name = str(groups_data[security_group_id]["name"])
+                products.add(product_name)
+                try:
+                    if len(product_name) > self.max_product_length:
+                        logger.error(
+                            "error inserting Bugzilla product and security group \"'%s' :: '%s'\" into db: product is too long (has %d characters, max is %d)",
+                            product_name,
+                            security_group_name,
+                            len(product_name),
+                            self.max_product_length,
+                        )
+                        continue
+                    if len(security_group_name) > self.max_security_group_length:
+                        logger.error(
+                            "error inserting Bugzilla product and security group \"'%s' :: '%s'\" into db: security group is too long (has %d characters, max is %d)",
+                            product_name,
+                            security_group_name,
+                            len(security_group_name),
+                            self.max_security_group_length,
+                        )
+                        continue
+                    BugzillaSecurityGroup.objects.get_or_create(
+                        product=product_name,
+                        security_group=security_group_name,
+                    )
+                except Exception as e:
+                    logger.error(
+                        "error inserting Bugzilla product and security group \"'%s' :: '%s'\" into db: %s",
+                        product_name,
+                        security_group_name,
+                        e,
+                    )
+                    continue
+        BugzillaSecurityGroup.objects.exclude(product__in=products).delete()

treeherder/etl/management/commands/update_bugzilla_security_groups.py

@@ -0,0 +1,11 @@
+from django.core.management.base import BaseCommand
+
+from treeherder.etl.files_bugzilla_map import ProductSecurityGroupProcess
+
+
+class Command(BaseCommand):
+    """Management command to manually update security groups for bugzilla products"""
+
+    def handle(self, *args, **options):
+        process = ProductSecurityGroupProcess()
+        process.run()

treeherder/model/migrations/0024_add_bugzillasecuritygroup.py

@@ -0,0 +1,30 @@
+# Generated by Django 3.1.12 on 2021-09-29 12:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('model', '0023_add_filebugzillacomponent'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='BugzillaSecurityGroup',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name='ID'
+                    ),
+                ),
+                ('product', models.CharField(db_index=True, max_length=60, unique=True)),
+                ('security_group', models.CharField(max_length=60)),
+            ],
+            options={
+                'verbose_name_plural': 'bugzilla_security_groups',
+                'db_table': 'bugzilla_security_group',
+            },
+        ),
+    ]

treeherder/model/models.py

@@ -360,6 +360,15 @@ class FilesBugzillaMap(models.Model):
         return "{0}".format(self.path)
 
 
+class BugzillaSecurityGroup(models.Model):
+    product = models.CharField(max_length=60, unique=True, db_index=True)
+    security_group = models.CharField(max_length=60)
+
+    class Meta:
+        db_table = 'bugzilla_security_group'
+        verbose_name_plural = 'bugzilla_security_groups'
+
+
 class Machine(NamedModel):
     class Meta:
         db_table = 'machine'

treeherder/webapp/api/bugzilla.py

@@ -7,6 +7,7 @@ from rest_framework.decorators import action
 from rest_framework.response import Response
 from rest_framework.status import HTTP_400_BAD_REQUEST
 
+from treeherder.model.models import BugzillaSecurityGroup
 from treeherder.utils.http import make_request
 
 
@@ -51,8 +52,20 @@ class BugzillaViewSet(viewsets.ViewSet):
             'description': description,
             'comment_tags': "treeherder",
         }
-        if len(params.get('groups')) > 0:
-            data['groups'] = params.get('groups')
+        if params.get("is_security_issue"):
+            security_group_list = list(
+                BugzillaSecurityGroup.objects.filter(product=data.get("product")).values_list(
+                    "security_group", flat=True
+                )
+            )
+            if len(security_group_list) == 0:
+                return Response(
+                    {
+                        "failure": "Cannot file security bug for product without default security group in Bugzilla."
+                    },
+                    status=HTTP_400_BAD_REQUEST,
+                )
+            data["groups"] = security_group_list
 
         try:
             response = make_request(url, method='POST', headers=headers, json=data)

ui/helpers/http.js

@@ -21,7 +21,7 @@ export const getData = async function getData(url, options = {}) {
   const contentType =
     response.headers.get('content-type') || ''.startsWith('text/html');
 
-  if (contentType && failureStatus) {
+  if (contentType && contentType !== 'application/json' && failureStatus) {
     const errorMessage = processErrorMessage(
       `${failureStatus}: ${response.statusText}`,
       failureStatus,

ui/shared/BugFiler.jsx

@@ -441,7 +441,7 @@ export class BugFilerClass extends React.Component {
           crash_signature: crashSignature,
           severity: 'S4',
           priority,
-          groups: isSecurityIssue ? ['core-security'] : [],
+          is_security_issue: isSecurityIssue,
           comment: descriptionStrings,
           comment_tags: 'treeherder',
         };
@@ -473,7 +473,7 @@ export class BugFilerClass extends React.Component {
   submitFailure = (source, status, statusText, data) => {
     const { notify } = this.props;
 
-    let failureString = `${source} returned status ${status}(${statusText})`;
+    let failureString = `${source} returned status ${status} (${statusText})`;
     if (data && data.failure) {
       failureString += `\n\n${data.failure}`;
     }