From 9cf09d8372f69e1999f457fc1cfa646944b53f8e Mon Sep 17 00:00:00 2001 From: Cameron Dawson Date: Tue, 7 Oct 2014 13:32:50 -0700 Subject: [PATCH] bug 1077136 - test passing for ingestion --- Vagrantfile | 4 +- tests/etl/test_buildapi.py | 51 +++++++++++--------- tests/sample_data/builds-pending-missing1.js | 2 +- treeherder/etl/cleanup_tasks.py | 4 -- 4 files changed, 32 insertions(+), 29 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index f0e5d0330..fd490618c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -8,7 +8,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # Every Vagrant virtual environment requires a box to build off of. config.vm.box = "precise32" config.vm.box_url = "http://files.vagrantup.com/precise32.box" - + config.vm.network "private_network", ip: "192.168.33.10" config.vm.synced_folder ".", "/home/vagrant/treeherder-service", type: "nfs" @@ -21,7 +21,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.provision :puppet do |puppet| puppet.manifests_path = "puppet/manifests" puppet.manifest_file = "vagrant.pp" - + #uncomment in production to serve treeherder-ui from dist directory #puppet.manifest_file = "production.pp" diff --git a/tests/etl/test_buildapi.py b/tests/etl/test_buildapi.py index ee44e686d..51bed6481 100644 --- a/tests/etl/test_buildapi.py +++ b/tests/etl/test_buildapi.py @@ -1,6 +1,7 @@ import os import pytest import responses +import json from django.conf import settings @@ -78,38 +79,39 @@ def test_ingest_pending_jobs(jm, initial_data, assert len(stored_obj) == 1 -def test_ingest_pending_jobs_1_missing_resultset(jm, initial_data, +def test_ingest_pending_jobs_1_missing_resultset(jm, initial_data, sample_resultset, test_repository, - test_base_dir, mock_buildapi_pending_missing1_url, mock_post_json_data, - mock_log_parser, mock_get_resultset, mock_get_remote_content, activate_responses): """ Ensure the job with the missing resultset is queued for refetching """ - pushlog_content = {"33270": { - "date": 1378288232, - "changesets": [ - { - "node": "4b40022e5c4cb344655ed7be9a408d2970a736c4", - "files": [ - "browser/base/content/browser.js" - ], - "tags": [], - "author": "John Doe ", - "branch": "default", - "desc": "bug 909264 - control characters in the location bar " - "should be %-encoded for visibility. r=gavin" - } - ], - "user": "jdoe@mozilla.com" - }} - pushlog_fake_url = "https://hg.mozilla.org/mozilla-central/json-pushes/?full=1&changeset=4b40022e5c4c" + new_revision = '222222222222' + pushlog_content = json.dumps( + {"33270": { + "date": 1378288232, + "changesets": [ + { + "node": new_revision + "b344655ed7be9a408d2970a736c4", + "files": [ + "browser/base/content/browser.js" + ], + "tags": [], + "author": "John Doe ", + "branch": "default", + "desc": "bug 909264 - control characters" + } + ], + "user": "jdoe@mozilla.com" + }} + ) + pushlog_fake_url = "https://hg.mozilla.org/mozilla-central/json-pushes/?full=1&changeset=" + new_revision responses.add(responses.GET, pushlog_fake_url, body=pushlog_content, status=200, + match_querystring=True, content_type='application/json') from treeherder.etl.buildapi import PendingJobsProcess @@ -126,7 +128,12 @@ def test_ingest_pending_jobs_1_missing_resultset(jm, initial_data, return_type='tuple' ) - assert len(revisions_stored) == 15 + assert len(revisions_stored) == 20 + was_stored = False + for rs in revisions_stored: + if str(rs['revision']) == new_revision: + was_stored = True + assert was_stored jm.disconnect() diff --git a/tests/sample_data/builds-pending-missing1.js b/tests/sample_data/builds-pending-missing1.js index c264c7132..cf6dc76c1 100644 --- a/tests/sample_data/builds-pending-missing1.js +++ b/tests/sample_data/builds-pending-missing1.js @@ -1,7 +1,7 @@ { "pending":{ "test_treeherder": { - "4b40022e5c4c": [ + "45f8637cb9f7": [ {"submitted_at": 1369304814, "id": 24575179, "buildername": "WINNT 6.2 try debug test mochitest-4"} ], "222222222222": [ diff --git a/treeherder/etl/cleanup_tasks.py b/treeherder/etl/cleanup_tasks.py index 03f71518e..ae474ad88 100644 --- a/treeherder/etl/cleanup_tasks.py +++ b/treeherder/etl/cleanup_tasks.py @@ -36,10 +36,6 @@ def fetch_missing_hg_push_logs(repo_name, repo_url, revisions): changesetParam = urllib.urlencode({"changeset": revisions}, True) urlStr = repo_url + '/json-pushes/?full=1&' + changesetParam - print "<><>" - print revisions - print urlStr - process.run(urlStr, repo_name)