Revert "Bug 1331399 - Trim job data that we store"

This reverts commit 613d7f0172.

tests/ui/job-view/PushList_test.jsx

@@ -19,7 +19,6 @@ import jobListFixtureOne from '../mock/job_list/job_1';
 import jobListFixtureTwo from '../mock/job_list/job_2';
 import configureStore from '../../../ui/job-view/redux/configureStore';
 import PushList from '../../../ui/job-view/pushes/PushList';
-import { getApiUrl } from '../../../ui/helpers/url';
 
 describe('PushList', () => {
   const repoName = 'autoland';
@@ -87,12 +86,18 @@ describe('PushList', () => {
       },
     );
     fetchMock.get(
-      getApiUrl('/jobs/?push_id=511138', repoName),
+      getProjectUrl(
+        '/jobs/?push_id=511138&count=2000&return_type=list',
+        repoName,
+      ),
       jobListFixtureOne,
     );
 
     fetchMock.mock(
-      getApiUrl('/jobs/?push_id=511137', repoName),
+      getProjectUrl(
+        '/jobs/?push_id=511137&count=2000&return_type=list',
+        repoName,
+      ),
       jobListFixtureTwo,
     );
   });

tests/ui/job-view/stores/pushes_test.jsx

@@ -31,7 +31,6 @@ import {
   updateRange,
 } from '../../../../ui/job-view/redux/stores/pushes';
 import { addAggregateFields } from '../../../../ui/helpers/job';
-import { getApiUrl } from '../../../../ui/helpers/url';
 
 const mockStore = configureMockStore([thunk]);
 
@@ -53,8 +52,15 @@ describe('Pushes Redux store', () => {
       getProjectUrl('/push/?full=true&count=10', repoName),
       pushListFixture,
     );
-    fetchMock.get(getApiUrl('/jobs/?push_id=1', repoName), jobListFixtureOne);
-    fetchMock.mock(getApiUrl('/jobs/?push_id=2', repoName), jobListFixtureTwo);
+    fetchMock.get(
+      getProjectUrl('/jobs/?push_id=1&count=2000&return_type=list', repoName),
+      jobListFixtureOne,
+    );
+
+    fetchMock.mock(
+      getProjectUrl('/jobs/?push_id=2&count=2000&return_type=list', repoName),
+      jobListFixtureTwo,
+    );
     const store = mockStore({ pushes: initialState });
 
     await store.dispatch(fetchPushes());
@@ -82,7 +88,7 @@ describe('Pushes Redux store', () => {
       pollPushListFixture,
     );
     fetchMock.mock(
-      `begin:${getApiUrl(
+      `begin:${getProjectUrl(
         '/jobs/?push_id__in=511138&last_modified__gt',
         repoName,
       )}`,

tests/ui/mock/job_list/pagination/page_1.json

@@ -1,61 +1,106 @@
 {
-  "count": 2,
-  "next": "/jobs/?push_id=526443&page=2",
-  "previous": null,
+  "meta": {
+    "count": 2,
+    "repository": "mozilla-central",
+    "offset": 0
+  },
+  "job_property_names": [
+    "submit_timestamp",
+    "machine_name",
+    "job_group_symbol",
+    "job_group_name",
+    "platform_option",
+    "job_type_description",
+    "result_set_id",
+    "result",
+    "id",
+    "machine_platform_architecture",
+    "end_timestamp",
+    "build_platform",
+    "job_guid",
+    "job_type_name",
+    "ref_data_name",
+    "platform",
+    "state",
+    "build_os",
+    "option_collection_hash",
+    "who",
+    "failure_classification_id",
+    "job_type_symbol",
+    "reason",
+    "job_group_description",
+    "machine_platform_os",
+    "start_timestamp",
+    "build_architecture",
+    "last_modified",
+    "build_platform_id"
+  ],
   "results": [
     [
-      2,
-      1,
-      259537193,
+      1424270698,
       "unknown",
       "?",
-      "Gecko Decision Task",
-      "D",
-      "2019-08-05T20:19:51.818175",
-      "102210fe594ee9b33d82058545b1ed14f4c8206e",
-      "gecko-decision",
+      "unknown",
       "opt",
-      526443,
-      "success",
-      "2aa083621bb989d6acf1151667288d5fe9616178",
-      "completed",
-      1
+      "",
+      1,
+      "unknown",
+      1,
+      "x86",
+      0,
+      "linux32",
+      "effdbbfd128cbe4a29e8d418230d708eea165c94",
+      "Build",
+      "Linux mozilla-inbound build",
+      "linux32",
+      "running",
+      216,
+      "linux",
+      "102210fe594ee9b33d82058545b1ed14f4c8206e",
+      "unknown",
+      1,
+      "B",
+      "unknown",
+      "",
+      null,
+      "linux",
+      1424271878,
+      "x86",
+      "2015-01-18T15:06:03",
+      17
     ],
     [
-      18,
-      1,
-      259537372,
-      "unknown",
+      1424270698,
+      "bld-linux64-spot-423",
       "?",
-      "build-android-api-16/debug",
-      "B",
-      "2019-08-05T20:37:45.258535",
-      "32faaecac742100f7753f0c1d0aa0add01b4046b",
-      "android-4-0-armv7-api16",
+      "unknown",
       "debug",
-      526443,
+      "",
+      1,
       "success",
-      "e2e58eb7f14af83019d3f8433c0642dde8082c4d",
+      2,
+      "x86",
+      1424272397,
+      "linux32",
+      "c63409cac418eb147286dfde9f773de6b2ad14b0",
+      "Build",
+      "Linux mozilla-inbound leak test build",
+      "linux32",
       "completed",
-      1
+      209,
+      "linux",
+      "32faaecac742100f7753f0c1d0aa0add01b4046b",
+      "mozilla-inbound-firefox",
+      1,
+      "B",
+      "scheduler",
+      "",
+      null,
+      "linux",
+      1424270703,
+      "x86",
+      "2015-01-18T15:19:17",
+      17
     ]
-  ],
-  "job_property_names": [
-    "duration",
-    "failure_classification_id",
-    "id",
-    "job_group_name",
-    "job_group_symbol",
-    "job_type_name",
-    "job_type_symbol",
-    "last_modified",
-    "option_collection_hash",
-    "platform",
-    "platform_option",
-    "push_id",
-    "result",
-    "signature",
-    "state",
-    "tier"
   ]
 }

tests/ui/mock/job_list/pagination/page_2.json

@@ -1,43 +1,73 @@
 {
-  "count": 1,
-  "next": null,
-  "previous": null,
+  "meta": {
+    "count": 2,
+    "repository": "mozilla-central",
+    "offset": 2
+  },
+  "job_property_names": [
+    "submit_timestamp",
+    "machine_name",
+    "job_group_symbol",
+    "job_group_name",
+    "platform_option",
+    "job_type_description",
+    "result_set_id",
+    "result",
+    "id",
+    "machine_platform_architecture",
+    "end_timestamp",
+    "build_platform",
+    "job_guid",
+    "job_type_name",
+    "ref_data_name",
+    "platform",
+    "state",
+    "build_os",
+    "option_collection_hash",
+    "who",
+    "failure_classification_id",
+    "job_type_symbol",
+    "reason",
+    "job_group_description",
+    "machine_platform_os",
+    "start_timestamp",
+    "build_architecture",
+    "last_modified",
+    "build_platform_id"
+  ],
   "results": [
     [
-      1,
-      1,
-      259539688,
+      1424270698,
       "unknown",
       "?",
-      "test-android-hw-p2-8-0-arm7-api-16/debug-fennec-jittest-1proc-8",
-      "Jit8",
-      "2019-08-05T20:57:09.299269",
-      "32faaecac742100f7753f0c1d0aa0add01b4046b",
-      "android-hw-p2-8-0-arm7-api-16",
-      "debug",
-      526443,
       "unknown",
-      "33ba86f5b1d8ad61599cb04b8d1f50b97fe19379",
+      "opt",
+      "",
+      1,
+      "unknown",
+      3,
+      "x86",
+      0,
+      "linux32",
+      "effdbbfd128cbe4a29e8d418230d708eea165c94",
+      "Build",
+      "Linux mozilla-inbound build",
+      "linux32",
       "running",
-      2
+      216,
+      "linux",
+      "102210fe594ee9b33d82058545b1ed14f4c8206e",
+      "unknown",
+      1,
+      "B",
+      "unknown",
+      "",
+      null,
+      "linux",
+      1424271878,
+      "x86",
+      "2015-01-18T15:06:03",
+      17
     ]
-  ],
-  "job_property_names": [
-    "duration",
-    "failure_classification_id",
-    "id",
-    "job_group_name",
-    "job_group_symbol",
-    "job_type_name",
-    "job_type_symbol",
-    "last_modified",
-    "option_collection_hash",
-    "platform",
-    "platform_option",
-    "push_id",
-    "result",
-    "signature",
-    "state",
-    "tier"
   ]
 }

tests/ui/models/jobs_test.js

@@ -1,7 +1,8 @@
 import { fetchMock } from 'fetch-mock';
 
 import JobModel from '../../../ui/models/job';
-import { getApiUrl } from '../../../ui/helpers/url';
+import { getProjectUrl } from '../../../ui/helpers/location';
+import jobListFixtureOne from '../mock/job_list/job_1';
 import paginatedJobListFixtureOne from '../mock/job_list/pagination/page_1';
 import paginatedJobListFixtureTwo from '../mock/job_list/pagination/page_2';
 
@@ -10,15 +11,25 @@ describe('JobModel', () => {
     fetchMock.reset();
   });
 
+  describe('getList', () => {
+    beforeEach(() => {
+      fetchMock.mock(getProjectUrl('/jobs/'), jobListFixtureOne);
+    });
+
+    test('should return a promise', () => {
+      const result = JobModel.getList();
+      expect(result.then).toBeDefined();
+    });
+  });
+
   describe('pagination', () => {
     beforeEach(() => {
-      fetchMock.mock(getApiUrl('/jobs/?count=2'), paginatedJobListFixtureOne);
       fetchMock.mock(
-        getApiUrl('/jobs/?push_id=526443'),
+        getProjectUrl('/jobs/?count=2'),
         paginatedJobListFixtureOne,
       );
       fetchMock.mock(
-        getApiUrl('/jobs/?push_id=526443&page=2'),
+        getProjectUrl('/jobs/?count=2&offset=2'),
         paginatedJobListFixtureTwo,
       );
     });
@@ -30,13 +41,10 @@ describe('JobModel', () => {
     });
 
     test('should return all the pages when fetchAll==true', async () => {
-      const { data } = await JobModel.getList(
-        { push_id: 526443 },
-        { fetchAll: true },
-      );
+      const { data } = await JobModel.getList({ count: 2 }, { fetchAll: true });
 
       expect(data).toHaveLength(3);
-      expect(data[2].id).toBe(259539688);
+      expect(data[2].id).toBe(3);
     });
   });
 });

treeherder/model/models.py

@@ -18,8 +18,7 @@ from django.db.utils import ProgrammingError
 from django.forms import model_to_dict
 from django.utils import timezone
 
-from treeherder.webapp.api.utils import (REPO_GROUPS,
-                                         to_timestamp)
+from treeherder.webapp.api.utils import REPO_GROUPS
 
 from ..services.elasticsearch import (bulk,
                                       index)
@@ -657,12 +656,6 @@ class Job(models.Model):
 
         return text_log_error
 
-    @staticmethod
-    def get_duration(submit_time, start_time, end_time):
-        endtime = end_time if to_timestamp(end_time) else datetime.datetime.now()
-        starttime = start_time if to_timestamp(start_time) else submit_time
-        return round((endtime - starttime).total_seconds() / 60)
-
 
 class TaskclusterMetadata(models.Model):
     '''

treeherder/webapp/api/jobs.py

@@ -96,73 +96,9 @@ class JobFilter(django_filters.FilterSet):
         }
 
 
-class JobsViewSet(viewsets.ReadOnlyModelViewSet):
+class JobsViewSet(viewsets.ViewSet):
     """
-    This viewset is the jobs endpoint.
-    """
-    _default_select_related = [
-        'job_type',
-        'job_group',
-        'machine_platform',
-        'signature',
-    ]
-    _query_field_names = [
-        'submit_time',
-        'start_time',
-        'end_time',
-        'failure_classification_id',
-        'id',
-        'job_group__name',
-        'job_group__symbol',
-        'job_type__name',
-        'job_type__symbol',
-        'last_modified',
-        'option_collection_hash',
-        'machine_platform__platform',
-        'option_collection_hash',
-        'push_id',
-        'result',
-        'signature__signature',
-        'state',
-        'tier',
-    ]
-    _output_field_names = [
-        'failure_classification_id',
-        'id',
-        'job_group_name',
-        'job_group_symbol',
-        'job_type_name',
-        'job_type_symbol',
-        'last_modified',
-        'platform',
-        'push_id',
-        'result',
-        'signature',
-        'state',
-        'tier',
-        'duration',
-        'platform_option',
-    ]
-    queryset = Job.objects.all().order_by('id').select_related(
-        *_default_select_related
-    ).values(*_query_field_names)
-    serializer_class = serializers.JobSerializer
-    filterset_class = JobFilter
-    pagination_class = pagination.JobPagination
-
-    def get_serializer_context(self):
-        option_collection_map = OptionCollection.objects.get_option_collection_map()
-        return {'option_collection_map': option_collection_map}
-
-    def list(self, request, *args, **kwargs):
-        resp = super().list(request, *args, **kwargs)
-        resp.data['job_property_names'] = self._output_field_names
-        return Response(resp.data)
-
-
-class JobsProjectViewSet(viewsets.ViewSet):
-    """
-    This viewset is the project bound version of the jobs endpoint.
+    This viewset is responsible for the jobs endpoint.
     """
 
     # data that we want to do select_related on when returning job objects
@@ -272,7 +208,7 @@ class JobsProjectViewSet(viewsets.ViewSet):
         except Job.DoesNotExist:
             return Response("No job with id: {0}".format(pk), status=HTTP_404_NOT_FOUND)
 
-        resp = serializers.JobProjectSerializer(job, read_only=True).data
+        resp = serializers.JobSerializer(job, read_only=True).data
 
         resp["resource_uri"] = reverse("jobs-detail",
                                        kwargs={"project": project, "pk": pk})

treeherder/webapp/api/pagination.py

@@ -1,26 +1,6 @@
-from django.core.paginator import Paginator
-from django.utils.functional import cached_property
 from rest_framework import pagination
 
 
 class IdPagination(pagination.CursorPagination):
     ordering = ('-id')
     page_size = 100
-
-
-# Django's Paginator class uses queryset.count() which
-# performs a full table scan
-class CustomPaginator(Paginator):
-    def __init__(self, queryset, page_size):
-        Paginator.__init__(self, queryset, page_size)
-
-    @cached_property
-    def count(self):
-        return len(self.object_list)
-
-
-class JobPagination(pagination.PageNumberPagination):
-    page_size = 2000
-    page_size_query_param = 'count'
-    max_page_size = 2000
-    django_paginator_class = CustomPaginator

treeherder/webapp/api/serializers.py

@@ -52,7 +52,7 @@ class TaskclusterMetadataSerializer(serializers.ModelSerializer):
         fields = '__all__'
 
 
-class JobProjectSerializer(serializers.ModelSerializer):
+class JobSerializer(serializers.ModelSerializer):
 
     def to_representation(self, job):
         return {
@@ -97,27 +97,6 @@ class JobProjectSerializer(serializers.ModelSerializer):
         fields = '__all__'
 
 
-class JobSerializer(serializers.ModelSerializer):
-
-    def to_representation(self, job):
-        option_collection_map = self.context['option_collection_map']
-        submit = job.pop('submit_time')
-        start = job.pop('start_time')
-        end = job.pop('end_time')
-        option_collection_hash = job.pop('option_collection_hash')
-
-        ret_val = list(job.values())
-        ret_val.extend([
-            models.Job.get_duration(submit, start, end),  # duration
-            option_collection_map.get(option_collection_hash, '')  # platform option
-        ])
-        return ret_val
-
-    class Meta:
-        model = models.Job
-        fields = '__all__'
-
-
 class FailureClassificationSerializer(serializers.ModelSerializer):
 
     class Meta:

treeherder/webapp/api/urls.py

@@ -20,15 +20,11 @@ from treeherder.webapp.api import (auth,
 
 # router for views that are bound to a project
 # i.e. all those views that don't involve reference data
-# DEPRECATED: We will be slowly transitioning away from this router
-# in favor of a router that does not require the ``project`` property.
 project_bound_router = routers.SimpleRouter()
 
-# DEPRECATED (in process): The UI is transitioning to the /jobs/ endpoint
-# from the default_router.
 project_bound_router.register(
     r'jobs',
-    jobs.JobsProjectViewSet,
+    jobs.JobsViewSet,
     base_name='jobs',
 )
 
@@ -95,10 +91,8 @@ tle_router.register(r'text-log-error',
                     text_log_error.TextLogErrorViewSet,
                     base_name='text-log-error')
 
-
 # refdata endpoints:
 default_router = routers.DefaultRouter()
-default_router.register(r'jobs', jobs.JobsViewSet, base_name='jobs')
 default_router.register(r'repository', refdata.RepositoryViewSet)
 default_router.register(r'taskclustermetadata', refdata.TaskclusterMetadataViewSet,
                         base_name='taskclustermetadata')

ui/helpers/job.js

@@ -237,15 +237,6 @@ export const addAggregateFields = function addAggregateFields(job) {
   return job;
 };
 
-// Remove any fields we don't need for display in the greater job field after
-// they have been mapped/processed.
-export const trim = function trim(job) {
-  delete job.option_collection_hash;
-  delete job.job_group_name;
-  delete job.job_type_name;
-  delete job.signature;
-};
-
 export const getJobSearchStrHref = function getJobSearchStrHref(jobSearchStr) {
   const params = getAllUrlParams();
   params.set('searchStr', jobSearchStr.split(' '));

ui/job-view/details/PinBoard.jsx

@@ -126,7 +126,10 @@ class PinBoard extends React.Component {
       return classification
         .create()
         .then(() => {
-          notify(`Classification saved for ${job.title}`, 'success');
+          notify(
+            `Classification saved for ${job.platform} ${job.job_type_name}`,
+            'success',
+          );
           // update the job to show that it's now classified
           const jobInstance = findJobInstance(job.id);
 

ui/job-view/pushes/Push.jsx

@@ -20,7 +20,6 @@ import {
   updateJobMap,
   recalculateUnclassifiedCounts,
 } from '../redux/stores/pushes';
-import { trim } from '../../helpers/job';
 
 import FuzzyJobFinder from './FuzzyJobFinder';
 import { Revision } from './Revision';
@@ -148,6 +147,8 @@ class Push extends React.PureComponent {
     const { data, failureStatus } = await JobModel.getList(
       {
         push_id: push.id,
+        count: 2000,
+        return_type: 'list',
       },
       { fetchAll: true },
     );
@@ -225,7 +226,6 @@ class Push extends React.PureComponent {
         platform.groups.push(group);
       }
       group.jobs.push(job);
-      trim(job);
     });
     return platforms;
   };

ui/models/job.js

@@ -20,20 +20,22 @@ export default class JobModel {
     // endpoint e.g. the similar jobs endpoint. It defaults to the job
     // list endpoint.
     const { fetchAll, uri: configUri } = config;
-    const jobUri = configUri || getApiUrl(uri);
+    const jobUri = configUri || getProjectUrl(uri);
     const { data, failureStatus } = await getData(
       `${jobUri}${options ? createQueryParams(options) : ''}`,
     );
 
     if (!failureStatus) {
-      const { results, job_property_names, next } = data;
+      const { results, meta, job_property_names } = data;
       let itemList;
       let nextPagesJobs = [];
 
-      // if ``next`` gives a URL for more, fetch the next pages
-      if (fetchAll && next) {
-        const page = new URLSearchParams(next.split('?')[1]).get('page');
-        const newOptions = { ...options, page };
+      // if the number of elements returned equals the page size,
+      // fetch the next pages
+      if (fetchAll && results.length === meta.count) {
+        const count = parseInt(meta.count, 10);
+        const offset = parseInt(meta.offset, 10) + count;
+        const newOptions = { ...options, offset, count };
         const {
           data: nextData,
           failureStatus: nextFailureStatus,