diff --git a/docker/entrypoint_prod.sh b/docker/entrypoint_prod.sh index 3b45aff8a..7362d7c12 100755 --- a/docker/entrypoint_prod.sh +++ b/docker/entrypoint_prod.sh @@ -66,6 +66,9 @@ elif [ "$1" == "update_bugscache" ]; then elif [ "$1" == "update_files_bugzilla_map" ]; then newrelic-admin run-program ./manage.py update_files_bugzilla_map +elif [ "$1" == "update_bugzilla_security_groups" ]; then + newrelic-admin run-program ./manage.py update_bugzilla_security_groups + elif [ "$1" == "cache_failure_history" ]; then newrelic-admin run-program ./manage.py cache_failure_history diff --git a/newrelic.ini b/newrelic.ini index 3ca08dce4..9c1eed145 100644 --- a/newrelic.ini +++ b/newrelic.ini @@ -30,4 +30,4 @@ shutdown_timeout = 15 # List finite-duration commands here to enable their annotation by the agent. # For infinite duration commands (such as `pulse_listener_*`) see: # https://docs.newrelic.com/docs/agents/python-agent/supported-features/python-background-tasks#wrapping -instrumentation.scripts.django_admin = update_changelog check cycle_data load_initial_data perf_sheriff report_backfill_outcome migrate update_bugscache update_files_bugzilla_map run_intermittents_commenter synthesize_backfill_report backfill_text_log_error_jobs +instrumentation.scripts.django_admin = update_changelog check cycle_data load_initial_data perf_sheriff report_backfill_outcome migrate update_bugscache update_files_bugzilla_map update_bugzilla_security_groups run_intermittents_commenter synthesize_backfill_report backfill_text_log_error_jobs diff --git a/tests/webapp/api/test_bugzilla.py b/tests/webapp/api/test_bugzilla.py index c782aec0a..8636242b3 100644 --- a/tests/webapp/api/test_bugzilla.py +++ b/tests/webapp/api/test_bugzilla.py @@ -50,7 +50,7 @@ def test_create_bug(client, eleven_jobs_stored, activate_responses, test_user): "comment": u"Intermittent Description", "comment_tags": "treeherder", "keywords": ["intermittent-failure"], - "groups": [], + "is_security_issue": False, }, ) assert resp.status_code == 200 @@ -103,7 +103,7 @@ def test_create_bug_with_unicode(client, eleven_jobs_stored, activate_responses, "comment": u"Intermittent “description” string", "comment_tags": "treeherder", "keywords": ["intermittent-failure"], - "groups": [], + "is_security_issue": False, }, ) assert resp.status_code == 200 @@ -158,7 +158,7 @@ def test_create_crash_bug(client, eleven_jobs_stored, activate_responses, test_u "crash_signature": "[@crashsig]", "priority": "--", "keywords": ["intermittent-failure", "crash"], - "groups": [], + "is_security_issue": False, }, ) assert resp.status_code == 200 @@ -207,7 +207,7 @@ def test_create_unauthenticated_bug(client, eleven_jobs_stored, activate_respons "comment_tags": "treeherder", "keywords": ["intermittent-failure"], "see_also": "12345", - "groups": [], + "is_security_issue": False, }, ) assert resp.status_code == 403 @@ -265,7 +265,7 @@ def test_create_bug_with_long_crash_signature( "crash_signature": crashsig, "regressed_by": "123", "see_also": "12345", - "groups": [], + "is_security_issue": False, }, ) assert resp.status_code == 400 diff --git a/treeherder/etl/files_bugzilla_map.py b/treeherder/etl/files_bugzilla_map.py index 556930a64..9d4599894 100644 --- a/treeherder/etl/files_bugzilla_map.py +++ b/treeherder/etl/files_bugzilla_map.py @@ -1,6 +1,12 @@ import logging +import sys -from treeherder.model.models import BugzillaComponent, FilesBugzillaMap, Repository +from treeherder.model.models import ( + BugzillaComponent, + BugzillaSecurityGroup, + FilesBugzillaMap, + Repository, +) from treeherder.utils.github import fetch_json logger = logging.getLogger(__name__) @@ -25,7 +31,7 @@ class FilesBugzillaMapProcess: component = product_component_data[1] if len(product) > self.max_product_length: logger.error( - "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): product is too long (has %d characters, max is %d", + "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): product is too long (has %d characters, max is %d)", product, component, path, @@ -35,17 +41,17 @@ class FilesBugzillaMapProcess: return if len(component) > self.max_component_length: logger.error( - "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): component is too long (has %d characters, max is %d", + "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): component is too long (has %d characters, max is %d)", product, component, path, - len(product), + len(component), self.max_component_length, ) return if len(path) > self.max_path_length: logger.error( - "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): path is too long (has %d characters, max is %d", + "error inserting Bugzilla product and component \"'%s' :: '%s'\" into db (file skipped: '%s'): path is too long (has %d characters, max is %d)", product, component, path, @@ -189,3 +195,73 @@ class FilesBugzillaMapProcess: ) bugzilla_components_unused = bugzilla_components_all.difference(bugzilla_components_used) (BugzillaComponent.objects.filter(id__in=bugzilla_components_unused).delete()) + + +class ProductSecurityGroupProcess: + max_product_length = BugzillaSecurityGroup._meta.get_field('product').max_length + max_security_group_length = BugzillaSecurityGroup._meta.get_field('security_group').max_length + + def fetch_data(self): + url = 'https://bugzilla.mozilla.org/latest/configuration' + product_security_group_data = None + exception = None + try: + product_security_group_data = fetch_json(url) + except Exception as e: + exception = e + return { + "url": url, + "product_security_group_data": product_security_group_data, + "exception": exception, + } + + def run(self): + data_returned = self.fetch_data() + if data_returned["exception"] is not None: + logger.error( + "error fetching file with map of source paths to Bugzilla products and components: url: %s ; %s", + data_returned["url"], + data_returned["exception"], + ) + sys.exit() + fields_data = data_returned["product_security_group_data"]["field"]["product"]["values"] + groups_data = data_returned["product_security_group_data"]["group"] + products = set() + for field_data in fields_data: + product_name = str(field_data["name"]) + security_group_id = str(field_data["security_group_id"]) + if security_group_id in groups_data: + security_group_name = str(groups_data[security_group_id]["name"]) + products.add(product_name) + try: + if len(product_name) > self.max_product_length: + logger.error( + "error inserting Bugzilla product and security group \"'%s' :: '%s'\" into db: product is too long (has %d characters, max is %d)", + product_name, + security_group_name, + len(product_name), + self.max_product_length, + ) + continue + if len(security_group_name) > self.max_security_group_length: + logger.error( + "error inserting Bugzilla product and security group \"'%s' :: '%s'\" into db: security group is too long (has %d characters, max is %d)", + product_name, + security_group_name, + len(security_group_name), + self.max_security_group_length, + ) + continue + BugzillaSecurityGroup.objects.get_or_create( + product=product_name, + security_group=security_group_name, + ) + except Exception as e: + logger.error( + "error inserting Bugzilla product and security group \"'%s' :: '%s'\" into db: %s", + product_name, + security_group_name, + e, + ) + continue + BugzillaSecurityGroup.objects.exclude(product__in=products).delete() diff --git a/treeherder/etl/management/commands/update_bugzilla_security_groups.py b/treeherder/etl/management/commands/update_bugzilla_security_groups.py new file mode 100644 index 000000000..0620ff551 --- /dev/null +++ b/treeherder/etl/management/commands/update_bugzilla_security_groups.py @@ -0,0 +1,11 @@ +from django.core.management.base import BaseCommand + +from treeherder.etl.files_bugzilla_map import ProductSecurityGroupProcess + + +class Command(BaseCommand): + """Management command to manually update security groups for bugzilla products""" + + def handle(self, *args, **options): + process = ProductSecurityGroupProcess() + process.run() diff --git a/treeherder/model/migrations/0024_add_bugzillasecuritygroup.py b/treeherder/model/migrations/0024_add_bugzillasecuritygroup.py new file mode 100644 index 000000000..9e60bd79a --- /dev/null +++ b/treeherder/model/migrations/0024_add_bugzillasecuritygroup.py @@ -0,0 +1,30 @@ +# Generated by Django 3.1.12 on 2021-09-29 12:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('model', '0023_add_filebugzillacomponent'), + ] + + operations = [ + migrations.CreateModel( + name='BugzillaSecurityGroup', + fields=[ + ( + 'id', + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name='ID' + ), + ), + ('product', models.CharField(db_index=True, max_length=60, unique=True)), + ('security_group', models.CharField(max_length=60)), + ], + options={ + 'verbose_name_plural': 'bugzilla_security_groups', + 'db_table': 'bugzilla_security_group', + }, + ), + ] diff --git a/treeherder/model/models.py b/treeherder/model/models.py index aa53e24ea..cc711de92 100644 --- a/treeherder/model/models.py +++ b/treeherder/model/models.py @@ -360,6 +360,15 @@ class FilesBugzillaMap(models.Model): return "{0}".format(self.path) +class BugzillaSecurityGroup(models.Model): + product = models.CharField(max_length=60, unique=True, db_index=True) + security_group = models.CharField(max_length=60) + + class Meta: + db_table = 'bugzilla_security_group' + verbose_name_plural = 'bugzilla_security_groups' + + class Machine(NamedModel): class Meta: db_table = 'machine' diff --git a/treeherder/webapp/api/bugzilla.py b/treeherder/webapp/api/bugzilla.py index 6ea8beffc..986367ace 100644 --- a/treeherder/webapp/api/bugzilla.py +++ b/treeherder/webapp/api/bugzilla.py @@ -7,6 +7,7 @@ from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.status import HTTP_400_BAD_REQUEST +from treeherder.model.models import BugzillaSecurityGroup from treeherder.utils.http import make_request @@ -51,8 +52,20 @@ class BugzillaViewSet(viewsets.ViewSet): 'description': description, 'comment_tags': "treeherder", } - if len(params.get('groups')) > 0: - data['groups'] = params.get('groups') + if params.get("is_security_issue"): + security_group_list = list( + BugzillaSecurityGroup.objects.filter(product=data.get("product")).values_list( + "security_group", flat=True + ) + ) + if len(security_group_list) == 0: + return Response( + { + "failure": "Cannot file security bug for product without default security group in Bugzilla." + }, + status=HTTP_400_BAD_REQUEST, + ) + data["groups"] = security_group_list try: response = make_request(url, method='POST', headers=headers, json=data) diff --git a/ui/helpers/http.js b/ui/helpers/http.js index 38edc1439..d3e8b3f99 100644 --- a/ui/helpers/http.js +++ b/ui/helpers/http.js @@ -21,7 +21,7 @@ export const getData = async function getData(url, options = {}) { const contentType = response.headers.get('content-type') || ''.startsWith('text/html'); - if (contentType && failureStatus) { + if (contentType && contentType !== 'application/json' && failureStatus) { const errorMessage = processErrorMessage( `${failureStatus}: ${response.statusText}`, failureStatus, diff --git a/ui/shared/BugFiler.jsx b/ui/shared/BugFiler.jsx index faa8fbaaf..aa18640d2 100644 --- a/ui/shared/BugFiler.jsx +++ b/ui/shared/BugFiler.jsx @@ -441,7 +441,7 @@ export class BugFilerClass extends React.Component { crash_signature: crashSignature, severity: 'S4', priority, - groups: isSecurityIssue ? ['core-security'] : [], + is_security_issue: isSecurityIssue, comment: descriptionStrings, comment_tags: 'treeherder', }; @@ -473,7 +473,7 @@ export class BugFilerClass extends React.Component { submitFailure = (source, status, statusText, data) => { const { notify } = this.props; - let failureString = `${source} returned status ${status}(${statusText})`; + let failureString = `${source} returned status ${status} (${statusText})`; if (data && data.failure) { failureString += `\n\n${data.failure}`; }