mozilla
/
treeherder
зеркало из https://github.com/mozilla/treeherder.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1178234 - Remove repository_version 

Since it's unused, and hg.mozilla.org now has this information available
via its API.

Note: This commit depends on bug 1178719, to prevent issues during
deployment. Also, due to https://code.djangoproject.com/ticket/25036 a
migrate will need to be run interactively after deployment, to clean up
the old repositoryversion content type.
This commit is contained in:
Ed Morley 2015-06-30 03:41:22 +01:00
Родитель caaec5d047
Коммит 6b1e4f602b
11 изменённых файлов: 1 добавлений и 361 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

125
tests/model/derived/test_refdata.py
Просмотреть файл

@ -4,28 +4,12 @@
import os
import time
import urllib2
import json
from datetime import datetime, timedelta
import pytest
from mock import Mock
from django.core.management import call_command
from treeherder.model.models import (RepositoryGroup,
Repository, RepositoryVersion)
@pytest.fixture
def mock_urllib():
"""this mocks urllib to avoid hitting the network
when retrieving the hg version file"""
mock = Mock()
mock.return_value = (
'#just comments',
'latest version'
)
urllib2.urlopen = mock
from treeherder.model.models import RepositoryGroup, Repository
@pytest.fixture
@ -44,21 +28,6 @@ def repository_id():
return repo.id
@pytest.fixture
def old_version_repository(repository_id):
version = RepositoryVersion.objects.create(repository_id=repository_id,
version='1.0',
version_timestamp=time.time())
return repository_id, version.id
@pytest.fixture
def latest_version_repository(repository_id):
version = RepositoryVersion.objects.create(repository_id=repository_id,
version='latest version',
version_timestamp=time.time())
return repository_id, version.id
time_now = int(time.time())
test_params = [
{
@ -305,25 +274,6 @@ def test_add_job_type(refdata):
assert row_data == expected
def test_get_or_create_repository_version(refdata, repository_id):
id = refdata.get_or_create_repository_version(
repository_id, 'v1.0', 1367248930.235682)
row_data = refdata.dhub.execute(
proc='refdata_test.selects.test_repository_version',
placeholders=[id],
return_type='iter'
)
refdata.disconnect()
assert row_data.get_column_data('repository_id') == repository_id
assert row_data.get_column_data('version') == 'v1.0'
assert row_data.get_column_data('version_timestamp') == 1367248930
assert row_data.get_column_data('active_status') == 'active'
def test_get_repository_info(refdata, repository_id):
"""test get_repository_info retrieves the right informations"""
@ -345,79 +295,6 @@ def test_get_repository_info(refdata, repository_id):
assert info[k] == v
def test_get_hg_repository_version(refdata, mock_urllib):
version = refdata.get_hg_repository_version("https://hg.mozilla.org/mozilla-central")
refdata.disconnect()
assert version == 'latest version'
def test_update_repo_version_if_old(refdata, old_version_repository, mock_urllib):
"""test repo version is updated if a new one is available"""
repo_id, old_version = old_version_repository
refdata.update_repository_version(repo_id)
updated_version = refdata.get_repository_version_id(repo_id)
refdata.disconnect()
assert old_version != updated_version
def test_update_repo_version_unchanged(refdata, latest_version_repository, mock_urllib):
"""Test version is kept and version_timestamp updated if the version is unchanged."""
time_now = time.time()
repo_id, last_version = latest_version_repository
refdata.update_repository_version(repo_id)
updated_version = refdata.get_repository_version_id(repo_id)
row_data = refdata.dhub.execute(
proc='refdata_test.selects.test_repository_version',
placeholders=[updated_version],
return_type='iter'
)
refdata.disconnect()
assert row_data.get_column_data('version') == 'latest version'
assert row_data.get_column_data('version_timestamp') >= long(time_now)
def test_update_repo_version_command(refdata, old_version_repository, initial_data, mock_urllib):
"""Test the django command extension update_repository_version without filters."""
repo_id, old_version = old_version_repository
call_command('update_repository_version')
updated_version = refdata.get_repository_version_id(repo_id)
refdata.disconnect()
assert old_version < updated_version
def test_update_repo_version_command_with_filters(refdata, old_version_repository, initial_data, mock_urllib):
"""Test the django command extension update_repository_version using filters."""
repo_id, old_version = old_version_repository
call_command('update_repository_version',
repo_name='mozilla-central',
group_name='mygroup',
codebase='gecko')
updated_version = refdata.get_repository_version_id(repo_id)
refdata.disconnect()
assert old_version < updated_version
@pytest.fixture
def sample_bugs(test_base_dir):
filename = os.path.join(

7
tests/refdata_test.json
Просмотреть файл

@ -105,13 +105,6 @@
`id` = ?",
"host_type":"read_host"
},
"test_repository_version":{
"sql": "SELECT `repository_id`, `version`, `version_timestamp`, `active_status`
FROM `repository_version`
WHERE
`id` = ?",
"host_type":"read_host"
},
"test_bugscache": {
"sql": "SELECT * FROM `bugscache`",
"host_type": "read_host"

80
treeherder/model/derived/refdata.py
Просмотреть файл

@ -7,7 +7,6 @@ import logging
from hashlib import sha1
import time
from datetime import timedelta, datetime
import urllib2
from django.conf import settings
from datasource.bases.BaseHub import BaseHub
from datasource.DataHub import DataHub
@ -1218,85 +1217,6 @@ class RefDataManager(object):
return id_iter.get_column_data('id')
def get_repository_version_id(self, repository_id):
"""get the latest version available for the given repository"""
id_iter = self.execute(
proc='reference.selects.get_repository_version_id',
placeholders=[repository_id],
debug_show=self.DEBUG,
return_type='iter')
return id_iter.get_column_data('id')
def get_or_create_repository_version(self, repository_id, version,
version_timestamp):
self.execute(
proc='reference.inserts.create_repository_version',
placeholders=[
repository_id,
version,
version_timestamp,
repository_id,
version
],
debug_show=self.DEBUG)
return self.get_repository_version_id(repository_id)
def update_repository_version(self, repository_id):
"""update repository version with the latest information
avaliable. the only dvcs supported is hg and the repo must
have an active_status of 'active'."""
# Imported here since doing so globally results in a ImportError
# TODO: Fix dependency loop and move this out.
from treeherder.model.derived.base import ObjectNotFoundException
repository = self.get_repository_info(repository_id)
if not repository:
raise ObjectNotFoundException("repository", id=repository_id)
if repository['dvcs_type'] != 'hg':
# TODO: Add handling for git, for now do nothing and return
# raise NotImplementedError
return
else:
version = self.get_hg_repository_version(repository['url'])
timestamp_now = time.time()
# try to create a new repository version
self.get_or_create_repository_version(repository_id,
version, timestamp_now)
# update the version_timestamp
self.execute(
proc='reference.updates.update_version_timestamp',
placeholders=[
timestamp_now,
repository_id,
version
],
debug_show=self.DEBUG)
def get_hg_repository_version(self, repo_url):
"""retrieves the milestone.txt file used to indicate
the current milestone of a repo. the last line contains
the info needed"""
milestone_path = '/raw-file/default/config/milestone.txt'
version_url = "".join((repo_url, milestone_path))
response = urllib2.urlopen(
version_url,
timeout=settings.TREEHERDER_REQUESTS_TIMEOUT)
for line in response:
# go to the last line
pass
return line.strip()
def get_repository_info(self, repository_id):
"""retrieves all the attributes of a repository"""

47
treeherder/model/management/commands/update_repository_version.py
Просмотреть файл

@ -1,47 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at http://mozilla.org/MPL/2.0/.
from optparse import make_option
from django.core.management.base import BaseCommand
from treeherder.model.derived import RefDataManager
from treeherder.model.models import Repository
class Command(BaseCommand):
help = """Update the given repository version. Only hg repositories are supported for now and the repo must have an active_status of 'active'."""
option_list = BaseCommand.option_list + (
make_option('--group',
action='store',
dest='group',
default=None,
help='Filter the repositories to update by group name'),
make_option('--repo-name',
action='store',
dest='repo_name',
default=None,
help='Filter the repositories to update by name'),
make_option('--codebase',
action='store',
dest='codebase',
default=None,
help='Filter the repositories to update by codebase'),
)
def handle(self, *args, **options):
repositories = Repository.objects.filter(active_status='active')
if options['repo_name']:
repositories = repositories.filter(name=options['repo_name'])
if options['codebase']:
repositories = repositories.filter(codebase=options['codebase'])
if options['group']:
repositories = repositories.filter(
repository_group__name=options['group'])
repo_ids = repositories.values_list('id', flat=True)
refdata = RefDataManager()
for repo_id in repo_ids:
refdata.update_repository_version(repo_id)

14
treeherder/model/migrations/0001_initial.py
Просмотреть файл

@ -276,20 +276,6 @@ class Migration(migrations.Migration):
},
bases=(models.Model,),
),
migrations.CreateModel(
name='RepositoryVersion',
fields=[
('id', models.AutoField(serialize=False, primary_key=True)),
('version', models.CharField(max_length=50L)),
('version_timestamp', models.IntegerField()),
('active_status', models.CharField(default='active', max_length=7L, blank=True)),
('repository', models.ForeignKey(to='model.Repository')),
],
options={
'db_table': 'repository_version',
},
bases=(models.Model,),
),
migrations.CreateModel(
name='UserExclusionProfile',
fields=[

16
treeherder/model/models.py
Просмотреть файл

@ -403,22 +403,6 @@ class JobGroup(models.Model):
self.name, self.symbol)
@python_2_unicode_compatible
class RepositoryVersion(models.Model):
id = models.AutoField(primary_key=True)
repository = models.ForeignKey(Repository)
version = models.CharField(max_length=50L)
version_timestamp = models.IntegerField()
active_status = models.CharField(max_length=7L, blank=True, default='active')
class Meta:
db_table = 'repository_version'
def __str__(self):
return "{0} version {1}".format(
self.repository, self.version)
@python_2_unicode_compatible
class OptionCollection(models.Model):
id = models.AutoField(primary_key=True)

28
treeherder/model/sql/reference.json
Просмотреть файл

@ -123,18 +123,6 @@
)",
"host_type":"master_host"
},
"create_repository_version":{
"sql": "INSERT INTO `repository_version` (`repository_id`, `version`, `version_timestamp`)
SELECT ?, ?, ?
FROM DUAL
WHERE NOT EXISTS(
SELECT `repository_id`, `version`
FROM `repository_version`
WHERE `repository_id` = ? AND `version` = ?
)",
"host_type":"master_host"
},
"create_repository_group":{
"sql":"INSERT INTO `repository_group` (`name`, `description`)
SELECT ?, 'fill me'
@ -267,16 +255,6 @@
WHERE `name` = ?",
"host_type":"read_host"
},
"get_repository_version_id":{
"sql": "SELECT `id`
FROM `repository_version`
WHERE
`repository_id` = ?
AND `active_status` = 'active'
ORDER BY `id` DESC
LIMIT 0,1",
"host_type":"read_host"
},
"get_repository_info":{
"sql": "SELECT *
FROM `repository`
@ -346,12 +324,6 @@
}
},
"updates":{
"update_version_timestamp":{
"sql": "UPDATE `repository_version`
SET `version_timestamp` = ?
WHERE `repository_id` = ? AND `version` = ?",
"host_type":"master_host"
},
"update_machine_timestamp":{
"sql": "UPDATE `machine`
SET `last_timestamp` = ?

37
treeherder/model/sql/template_schema/treeherder_reference_1.sql.tmpl
Просмотреть файл

@ -594,43 +594,6 @@ CREATE TABLE `repository_group` (
) ENGINE={engine} DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `repository_version`
--
DROP TABLE IF EXISTS `repository_version`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
/**************************
* Table: repository_version
*
* List of versions associated with repositories. Current version will have the
* most recent timestamp associated with it.
*
* Population Method: This information could possibly be pulled automatically. It also
* might be coming in dynamically with test data.
*
* Example Data:
*
* repository_id - References repository.id
* version - 20.0a1 | 21.0a2 | ...
**************************/
CREATE TABLE `repository_version` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`repository_id` int(10) unsigned NOT NULL,
`version` varchar(50) COLLATE utf8_bin NOT NULL,
`version_timestamp` int(10) unsigned NOT NULL,
`active_status` enum('active','onhold','deleted') COLLATE utf8_bin DEFAULT 'active',
PRIMARY KEY (`id`),
KEY `idx_repository_id` (`repository_id`),
KEY `idx_version` (`version`),
KEY `idx_version_timestamp_id` (`version_timestamp`),
KEY `idx_active_status` (`active_status`),
CONSTRAINT `fk_repository` FOREIGN KEY (`repository_id`) REFERENCES `repository` (`id`)
) ENGINE={engine} DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
/*!40101 SET character_set_client = @saved_cs_client */;
DROP TABLE IF EXISTS `job_exclusion`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;

1
treeherder/webapp/admin.py
Просмотреть файл

@ -40,6 +40,5 @@ browserid_admin.register(Machine)
browserid_admin.register(MachineNote)
browserid_admin.register(Datasource)
browserid_admin.register(JobGroup)
browserid_admin.register(RepositoryVersion)
browserid_admin.register(OptionCollection)
browserid_admin.register(FailureClassification)

6
treeherder/webapp/api/refdata.py
Просмотреть файл

@ -85,12 +85,6 @@ class MachineNoteViewSet(viewsets.ReadOnlyModelViewSet):
model = models.MachineNote
class RepositoryVersionViewSet(viewsets.ReadOnlyModelViewSet):
"""ViewSet for the refdata RepositoryVersion model"""
model = models.RepositoryVersion
class OptionCollectionHashViewSet(viewsets.ViewSet):
"""ViewSet for the virtual OptionCollectionHash model"""

1
treeherder/webapp/api/urls.py
Просмотреть файл

@ -92,7 +92,6 @@ default_router.register(r'buildplatform', refdata.BuildPlatformViewSet)
default_router.register(r'jobgroup', refdata.JobGroupViewSet)
default_router.register(r'jobtype', refdata.JobTypeViewSet)
default_router.register(r'repository', refdata.RepositoryViewSet)
default_router.register(r'repositoryversion', refdata.RepositoryVersionViewSet)
default_router.register(r'optioncollectionhash', refdata.OptionCollectionHashViewSet,
base_name='optioncollectionhash')
default_router.register(r'bugscache', refdata.BugscacheViewSet)