diff --git a/tests/model/derived/test_refdata.py b/tests/model/derived/test_refdata.py index cdd8fd7ef..c1f870dd0 100644 --- a/tests/model/derived/test_refdata.py +++ b/tests/model/derived/test_refdata.py @@ -4,28 +4,12 @@ import os import time -import urllib2 import json from datetime import datetime, timedelta import pytest -from mock import Mock -from django.core.management import call_command -from treeherder.model.models import (RepositoryGroup, - Repository, RepositoryVersion) - - -@pytest.fixture -def mock_urllib(): - """this mocks urllib to avoid hitting the network - when retrieving the hg version file""" - mock = Mock() - mock.return_value = ( - '#just comments', - 'latest version' - ) - urllib2.urlopen = mock +from treeherder.model.models import RepositoryGroup, Repository @pytest.fixture @@ -44,21 +28,6 @@ def repository_id(): return repo.id -@pytest.fixture -def old_version_repository(repository_id): - version = RepositoryVersion.objects.create(repository_id=repository_id, - version='1.0', - version_timestamp=time.time()) - return repository_id, version.id - - -@pytest.fixture -def latest_version_repository(repository_id): - version = RepositoryVersion.objects.create(repository_id=repository_id, - version='latest version', - version_timestamp=time.time()) - return repository_id, version.id - time_now = int(time.time()) test_params = [ { @@ -305,25 +274,6 @@ def test_add_job_type(refdata): assert row_data == expected -def test_get_or_create_repository_version(refdata, repository_id): - - id = refdata.get_or_create_repository_version( - repository_id, 'v1.0', 1367248930.235682) - - row_data = refdata.dhub.execute( - proc='refdata_test.selects.test_repository_version', - placeholders=[id], - return_type='iter' - ) - - refdata.disconnect() - - assert row_data.get_column_data('repository_id') == repository_id - assert row_data.get_column_data('version') == 'v1.0' - assert row_data.get_column_data('version_timestamp') == 1367248930 - assert row_data.get_column_data('active_status') == 'active' - - def test_get_repository_info(refdata, repository_id): """test get_repository_info retrieves the right informations""" @@ -345,79 +295,6 @@ def test_get_repository_info(refdata, repository_id): assert info[k] == v -def test_get_hg_repository_version(refdata, mock_urllib): - version = refdata.get_hg_repository_version("https://hg.mozilla.org/mozilla-central") - - refdata.disconnect() - - assert version == 'latest version' - - -def test_update_repo_version_if_old(refdata, old_version_repository, mock_urllib): - """test repo version is updated if a new one is available""" - repo_id, old_version = old_version_repository - - refdata.update_repository_version(repo_id) - - updated_version = refdata.get_repository_version_id(repo_id) - - refdata.disconnect() - - assert old_version != updated_version - - -def test_update_repo_version_unchanged(refdata, latest_version_repository, mock_urllib): - """Test version is kept and version_timestamp updated if the version is unchanged.""" - - time_now = time.time() - repo_id, last_version = latest_version_repository - refdata.update_repository_version(repo_id) - - updated_version = refdata.get_repository_version_id(repo_id) - - row_data = refdata.dhub.execute( - proc='refdata_test.selects.test_repository_version', - placeholders=[updated_version], - return_type='iter' - ) - - refdata.disconnect() - - assert row_data.get_column_data('version') == 'latest version' - assert row_data.get_column_data('version_timestamp') >= long(time_now) - - -def test_update_repo_version_command(refdata, old_version_repository, initial_data, mock_urllib): - """Test the django command extension update_repository_version without filters.""" - - repo_id, old_version = old_version_repository - - call_command('update_repository_version') - - updated_version = refdata.get_repository_version_id(repo_id) - - refdata.disconnect() - - assert old_version < updated_version - - -def test_update_repo_version_command_with_filters(refdata, old_version_repository, initial_data, mock_urllib): - """Test the django command extension update_repository_version using filters.""" - - repo_id, old_version = old_version_repository - - call_command('update_repository_version', - repo_name='mozilla-central', - group_name='mygroup', - codebase='gecko') - - updated_version = refdata.get_repository_version_id(repo_id) - - refdata.disconnect() - - assert old_version < updated_version - - @pytest.fixture def sample_bugs(test_base_dir): filename = os.path.join( diff --git a/tests/refdata_test.json b/tests/refdata_test.json index 467955106..d79c23acb 100644 --- a/tests/refdata_test.json +++ b/tests/refdata_test.json @@ -105,13 +105,6 @@ `id` = ?", "host_type":"read_host" }, - "test_repository_version":{ - "sql": "SELECT `repository_id`, `version`, `version_timestamp`, `active_status` - FROM `repository_version` - WHERE - `id` = ?", - "host_type":"read_host" - }, "test_bugscache": { "sql": "SELECT * FROM `bugscache`", "host_type": "read_host" diff --git a/treeherder/model/derived/refdata.py b/treeherder/model/derived/refdata.py index d438a161a..58f7757fd 100644 --- a/treeherder/model/derived/refdata.py +++ b/treeherder/model/derived/refdata.py @@ -7,7 +7,6 @@ import logging from hashlib import sha1 import time from datetime import timedelta, datetime -import urllib2 from django.conf import settings from datasource.bases.BaseHub import BaseHub from datasource.DataHub import DataHub @@ -1218,85 +1217,6 @@ class RefDataManager(object): return id_iter.get_column_data('id') - def get_repository_version_id(self, repository_id): - """get the latest version available for the given repository""" - - id_iter = self.execute( - proc='reference.selects.get_repository_version_id', - placeholders=[repository_id], - debug_show=self.DEBUG, - return_type='iter') - - return id_iter.get_column_data('id') - - def get_or_create_repository_version(self, repository_id, version, - version_timestamp): - - self.execute( - proc='reference.inserts.create_repository_version', - placeholders=[ - repository_id, - version, - version_timestamp, - repository_id, - version - ], - debug_show=self.DEBUG) - - return self.get_repository_version_id(repository_id) - - def update_repository_version(self, repository_id): - """update repository version with the latest information - avaliable. the only dvcs supported is hg and the repo must - have an active_status of 'active'.""" - # Imported here since doing so globally results in a ImportError - # TODO: Fix dependency loop and move this out. - from treeherder.model.derived.base import ObjectNotFoundException - - repository = self.get_repository_info(repository_id) - - if not repository: - raise ObjectNotFoundException("repository", id=repository_id) - - if repository['dvcs_type'] != 'hg': - # TODO: Add handling for git, for now do nothing and return - # raise NotImplementedError - return - else: - version = self.get_hg_repository_version(repository['url']) - - timestamp_now = time.time() - - # try to create a new repository version - self.get_or_create_repository_version(repository_id, - version, timestamp_now) - - # update the version_timestamp - self.execute( - proc='reference.updates.update_version_timestamp', - placeholders=[ - timestamp_now, - repository_id, - version - ], - debug_show=self.DEBUG) - - def get_hg_repository_version(self, repo_url): - """retrieves the milestone.txt file used to indicate - the current milestone of a repo. the last line contains - the info needed""" - - milestone_path = '/raw-file/default/config/milestone.txt' - version_url = "".join((repo_url, milestone_path)) - - response = urllib2.urlopen( - version_url, - timeout=settings.TREEHERDER_REQUESTS_TIMEOUT) - for line in response: - # go to the last line - pass - return line.strip() - def get_repository_info(self, repository_id): """retrieves all the attributes of a repository""" diff --git a/treeherder/model/management/commands/update_repository_version.py b/treeherder/model/management/commands/update_repository_version.py deleted file mode 100644 index e88872239..000000000 --- a/treeherder/model/management/commands/update_repository_version.py +++ /dev/null @@ -1,47 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, you can obtain one at http://mozilla.org/MPL/2.0/. - -from optparse import make_option -from django.core.management.base import BaseCommand -from treeherder.model.derived import RefDataManager -from treeherder.model.models import Repository - - -class Command(BaseCommand): - help = """Update the given repository version. Only hg repositories are supported for now and the repo must have an active_status of 'active'.""" - - option_list = BaseCommand.option_list + ( - make_option('--group', - action='store', - dest='group', - default=None, - help='Filter the repositories to update by group name'), - make_option('--repo-name', - action='store', - dest='repo_name', - default=None, - help='Filter the repositories to update by name'), - make_option('--codebase', - action='store', - dest='codebase', - default=None, - help='Filter the repositories to update by codebase'), - ) - - def handle(self, *args, **options): - repositories = Repository.objects.filter(active_status='active') - if options['repo_name']: - repositories = repositories.filter(name=options['repo_name']) - if options['codebase']: - repositories = repositories.filter(codebase=options['codebase']) - if options['group']: - repositories = repositories.filter( - repository_group__name=options['group']) - - repo_ids = repositories.values_list('id', flat=True) - - refdata = RefDataManager() - - for repo_id in repo_ids: - refdata.update_repository_version(repo_id) diff --git a/treeherder/model/migrations/0001_initial.py b/treeherder/model/migrations/0001_initial.py index 842c4f933..1648818a5 100644 --- a/treeherder/model/migrations/0001_initial.py +++ b/treeherder/model/migrations/0001_initial.py @@ -276,20 +276,6 @@ class Migration(migrations.Migration): }, bases=(models.Model,), ), - migrations.CreateModel( - name='RepositoryVersion', - fields=[ - ('id', models.AutoField(serialize=False, primary_key=True)), - ('version', models.CharField(max_length=50L)), - ('version_timestamp', models.IntegerField()), - ('active_status', models.CharField(default='active', max_length=7L, blank=True)), - ('repository', models.ForeignKey(to='model.Repository')), - ], - options={ - 'db_table': 'repository_version', - }, - bases=(models.Model,), - ), migrations.CreateModel( name='UserExclusionProfile', fields=[ diff --git a/treeherder/model/models.py b/treeherder/model/models.py index e39682f05..ff6a25aa2 100644 --- a/treeherder/model/models.py +++ b/treeherder/model/models.py @@ -403,22 +403,6 @@ class JobGroup(models.Model): self.name, self.symbol) -@python_2_unicode_compatible -class RepositoryVersion(models.Model): - id = models.AutoField(primary_key=True) - repository = models.ForeignKey(Repository) - version = models.CharField(max_length=50L) - version_timestamp = models.IntegerField() - active_status = models.CharField(max_length=7L, blank=True, default='active') - - class Meta: - db_table = 'repository_version' - - def __str__(self): - return "{0} version {1}".format( - self.repository, self.version) - - @python_2_unicode_compatible class OptionCollection(models.Model): id = models.AutoField(primary_key=True) diff --git a/treeherder/model/sql/reference.json b/treeherder/model/sql/reference.json index 35a5e6141..de07e1b4f 100644 --- a/treeherder/model/sql/reference.json +++ b/treeherder/model/sql/reference.json @@ -123,18 +123,6 @@ )", "host_type":"master_host" }, - - "create_repository_version":{ - "sql": "INSERT INTO `repository_version` (`repository_id`, `version`, `version_timestamp`) - SELECT ?, ?, ? - FROM DUAL - WHERE NOT EXISTS( - SELECT `repository_id`, `version` - FROM `repository_version` - WHERE `repository_id` = ? AND `version` = ? - )", - "host_type":"master_host" - }, "create_repository_group":{ "sql":"INSERT INTO `repository_group` (`name`, `description`) SELECT ?, 'fill me' @@ -267,16 +255,6 @@ WHERE `name` = ?", "host_type":"read_host" }, - "get_repository_version_id":{ - "sql": "SELECT `id` - FROM `repository_version` - WHERE - `repository_id` = ? - AND `active_status` = 'active' - ORDER BY `id` DESC - LIMIT 0,1", - "host_type":"read_host" - }, "get_repository_info":{ "sql": "SELECT * FROM `repository` @@ -346,12 +324,6 @@ } }, "updates":{ - "update_version_timestamp":{ - "sql": "UPDATE `repository_version` - SET `version_timestamp` = ? - WHERE `repository_id` = ? AND `version` = ?", - "host_type":"master_host" - }, "update_machine_timestamp":{ "sql": "UPDATE `machine` SET `last_timestamp` = ? diff --git a/treeherder/model/sql/template_schema/treeherder_reference_1.sql.tmpl b/treeherder/model/sql/template_schema/treeherder_reference_1.sql.tmpl index 8526ad200..4fec26475 100644 --- a/treeherder/model/sql/template_schema/treeherder_reference_1.sql.tmpl +++ b/treeherder/model/sql/template_schema/treeherder_reference_1.sql.tmpl @@ -594,43 +594,6 @@ CREATE TABLE `repository_group` ( ) ENGINE={engine} DEFAULT CHARSET=utf8 COLLATE=utf8_bin; /*!40101 SET character_set_client = @saved_cs_client */; --- --- Table structure for table `repository_version` --- - -DROP TABLE IF EXISTS `repository_version`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; - -/************************** - * Table: repository_version - * - * List of versions associated with repositories. Current version will have the - * most recent timestamp associated with it. - * - * Population Method: This information could possibly be pulled automatically. It also - * might be coming in dynamically with test data. - * - * Example Data: - * - * repository_id - References repository.id - * version - 20.0a1 | 21.0a2 | ... - **************************/ -CREATE TABLE `repository_version` ( - `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `repository_id` int(10) unsigned NOT NULL, - `version` varchar(50) COLLATE utf8_bin NOT NULL, - `version_timestamp` int(10) unsigned NOT NULL, - `active_status` enum('active','onhold','deleted') COLLATE utf8_bin DEFAULT 'active', - PRIMARY KEY (`id`), - KEY `idx_repository_id` (`repository_id`), - KEY `idx_version` (`version`), - KEY `idx_version_timestamp_id` (`version_timestamp`), - KEY `idx_active_status` (`active_status`), - CONSTRAINT `fk_repository` FOREIGN KEY (`repository_id`) REFERENCES `repository` (`id`) -) ENGINE={engine} DEFAULT CHARSET=utf8 COLLATE=utf8_bin; -/*!40101 SET character_set_client = @saved_cs_client */; - DROP TABLE IF EXISTS `job_exclusion`; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; diff --git a/treeherder/webapp/admin.py b/treeherder/webapp/admin.py index 93fc99ad8..a9042d755 100644 --- a/treeherder/webapp/admin.py +++ b/treeherder/webapp/admin.py @@ -40,6 +40,5 @@ browserid_admin.register(Machine) browserid_admin.register(MachineNote) browserid_admin.register(Datasource) browserid_admin.register(JobGroup) -browserid_admin.register(RepositoryVersion) browserid_admin.register(OptionCollection) browserid_admin.register(FailureClassification) diff --git a/treeherder/webapp/api/refdata.py b/treeherder/webapp/api/refdata.py index 586ea7575..91ffdfc6a 100644 --- a/treeherder/webapp/api/refdata.py +++ b/treeherder/webapp/api/refdata.py @@ -85,12 +85,6 @@ class MachineNoteViewSet(viewsets.ReadOnlyModelViewSet): model = models.MachineNote -class RepositoryVersionViewSet(viewsets.ReadOnlyModelViewSet): - - """ViewSet for the refdata RepositoryVersion model""" - model = models.RepositoryVersion - - class OptionCollectionHashViewSet(viewsets.ViewSet): """ViewSet for the virtual OptionCollectionHash model""" diff --git a/treeherder/webapp/api/urls.py b/treeherder/webapp/api/urls.py index a3e445e6c..e9e75d7b4 100644 --- a/treeherder/webapp/api/urls.py +++ b/treeherder/webapp/api/urls.py @@ -92,7 +92,6 @@ default_router.register(r'buildplatform', refdata.BuildPlatformViewSet) default_router.register(r'jobgroup', refdata.JobGroupViewSet) default_router.register(r'jobtype', refdata.JobTypeViewSet) default_router.register(r'repository', refdata.RepositoryViewSet) -default_router.register(r'repositoryversion', refdata.RepositoryVersionViewSet) default_router.register(r'optioncollectionhash', refdata.OptionCollectionHashViewSet, base_name='optioncollectionhash') default_router.register(r'bugscache', refdata.BugscacheViewSet)