From da07781c744c9bafe5dd72ae46f5579b9bc30830 Mon Sep 17 00:00:00 2001 From: Shing Lyu Date: Wed, 30 Nov 2016 19:11:16 -0800 Subject: [PATCH] servo: Merge #14147 - Added an option to allow submitting test-perf result to perfherder (from shinglyu:test-perf-submit); r=larsbergstrom,aneeshusa This patch enables us to run `./mach test-perf --submit` in CI to submit the result to perfherder. r? @aneeshusa --- - [x] `./mach build -d` does not report any errors - [x] `./mach test-tidy` does not report any errors - [] These changes fix #__ (github issue number if applicable). - [ ] There are tests for these changes OR - [x] These changes do not require tests because too many manual setup required to test it Source-Repo: https://github.com/servo/servo Source-Revision: 7d66bd710cee081155487ef36fe5b490e181c780 --- servo/etc/ci/performance/README.md | 36 ++++++++------------------ servo/etc/ci/performance/test_all.sh | 2 +- servo/etc/ci/performance/test_perf.sh | 16 +++++++++--- servo/python/servo/testing_commands.py | 15 +++++++++-- 4 files changed, 38 insertions(+), 31 deletions(-) diff --git a/servo/etc/ci/performance/README.md b/servo/etc/ci/performance/README.md index b7baeb08453a..d65c79508b36 100644 --- a/servo/etc/ci/performance/README.md +++ b/servo/etc/ci/performance/README.md @@ -7,27 +7,21 @@ Servo Page Load Time Test # Basic Usage -## Prepare the test runner +`./mach test-perf` can be used to run a performance test on your servo build. The test result JSON will be saved to `etc/ci/performance/output/`. You can then run `python test_differ.py` to compare these two test results. Run `python test_differ.py -h` for instructions. + +# Setup for CI machine +## CI for Servo -* Clone this repo -* Download [tp5n.zip](http://people.mozilla.org/~jmaher/taloszips/zips/tp5n.zip), extract it to `page_load_test/tp5n` -* Run `prepare_manifest.sh` to transform the tp5n manifest to our format -* Install the Python3 `treeherder-client` package. For example, to install it in a virtualenv: `python3 -m virtualenv venv; source venv/bin/activate; pip install "treeherder-client>=3.0.0"` * Setup your Treeherder client ID and secret as environment variables `TREEHERDER_CLIENT_ID` and `TREEHERDER_CLIENT_SECRET` +* Run `./mach test-perf --submit` to run and submit the result to Perfherder. -## Build Servo -* Clone the servo repo -* Compile release build -* Run `git_log_to_json.sh` in the servo repo, save the output as `revision.json` -* Put your `servo` binary, `revision.json` and `resources` folder in `etc/ci/performance/servo/` +## CI for Gecko -## Run -* Activate the virutalenv: `source venv/bin/activate` -* Sync your system clock before running, the Perfherder API SSL check will fail if your system clock is not accurate. (e.g. `sudo nptdate tw.pool.ntp.org`) -* Run `test_all.sh [--servo|--gecko] [--submit]` - - choose `servo` or `gecko` as the testing engine - - enable `submit`, if you want to submit to perfherder -* Test results are submitted to https://treeherder.mozilla.org/#/jobs?repo=servo +* Install Firefox Nightly in your PATH +* Download [geckodriver](https://github.com/mozilla/geckodriver/releases) and add it to the `PATH` +* `pip install selenium` +* Run `python gecko_driver.py` to test +* Run `test_all.sh --gecko --submit` # How it works @@ -61,14 +55,6 @@ If you want to test the data submission code in `submit_to_perfherder.py` withou * `./manage.py create_credentials "description"`, the email has to match your logged in user. Remember to log-in through the Web UI once before you run this. * Setup your Treeherder client ID and secret as environment variables `TREEHERDER_CLIENT_ID` and `TREEHERDER_CLIENT_SECRET` -## For Gecko - -* Install Firefox Nightly in your PATH -* Download [geckodriver](https://github.com/mozilla/geckodriver/releases) and add it to the `PATH` -* `pip install selenium` -* Run `python gecko_driver.py` to test - - # Troubleshooting If you saw this error message: diff --git a/servo/etc/ci/performance/test_all.sh b/servo/etc/ci/performance/test_all.sh index 35031da0be0a..cedf1e60aa9b 100755 --- a/servo/etc/ci/performance/test_all.sh +++ b/servo/etc/ci/performance/test_all.sh @@ -57,7 +57,7 @@ then # results appear on the same date. Use the correct result when Perfherder # allows us to change the date. python3 submit_to_perfherder.py \ - "${output:-}" "${engine}" "${PERF_FILE}" servo/revision.json + "${engine}" "${PERF_FILE}" servo/revision.json fi echo "Stopping the local server" diff --git a/servo/etc/ci/performance/test_perf.sh b/servo/etc/ci/performance/test_perf.sh index 4b239b8831f4..fe76368fb4d4 100755 --- a/servo/etc/ci/performance/test_perf.sh +++ b/servo/etc/ci/performance/test_perf.sh @@ -32,6 +32,16 @@ PS1="" source venv/bin/activate pip install "treeherder-client>=3.0.0" mkdir -p servo -mkdir -p output -./git_log_to_json.sh > servo/revision.json && \ -./test_all.sh --servo +mkdir -p output # Test result will be saved to output/perf-.json +./git_log_to_json.sh > servo/revision.json + +if [[ "${#}" -eq 1 ]]; then + if [[ "${1}" = "--submit" ]]; then + ./test_all.sh --servo --submit + else + echo "Unrecognized argument: ${1}; Ignore and proceed without submitting" + ./test_all.sh --servo + fi +else + ./test_all.sh --servo +fi diff --git a/servo/python/servo/testing_commands.py b/servo/python/servo/testing_commands.py index a116a1eb8048..5331fc4b7575 100644 --- a/servo/python/servo/testing_commands.py +++ b/servo/python/servo/testing_commands.py @@ -166,12 +166,23 @@ class MachCommands(CommandBase): @Command('test-perf', description='Run the page load performance test', category='testing') - def test_perf(self): + @CommandArgument('--submit', default=False, action="store_true", + help="submit the data to perfherder") + def test_perf(self, submit=False): self.set_software_rendering_env(True) self.ensure_bootstrapped() env = self.build_env() - return call(["bash", "test_perf.sh"], + cmd = ["bash", "test_perf.sh"] + if submit: + if not ("TREEHERDER_CLIENT_ID" in os.environ and + "TREEHERDER_CLIENT_SECRET" in os.environ): + print("Please set the environment variable \"TREEHERDER_CLIENT_ID\"" + " and \"TREEHERDER_CLIENT_SECRET\" to submit the performance" + " test result to perfherder") + return 1 + cmd += ["--submit"] + return call(cmd, env=env, cwd=path.join("etc", "ci", "performance"))