Added stability to OSS instrumentation tests
Summary: This change makes all instrumentation tests to be executed in sequence in independent retriable processes. With a new test being open sourced recently our CI stability degraded. This PR should bring back stability because tests won't affect each other and will have shorter lifetime Closes https://github.com/facebook/react-native/pull/7353 Differential Revision: D3259081 fb-gh-sync-id: 48ccdb5dbd561d416526497ff474378db9ca3c60 fbshipit-source-id: 48ccdb5dbd561d416526497ff474378db9ca3c60
This commit is contained in:
Родитель
2760df761d
Коммит
110d1587ae
|
@ -6,7 +6,7 @@
|
||||||
* of patent rights can be found in the PATENTS file in the same directory.
|
* of patent rights can be found in the PATENTS file in the same directory.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package com.facebook.react;
|
package com.facebook.react.tests;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
|
@ -69,7 +69,7 @@ test:
|
||||||
# build test APK
|
# build test APK
|
||||||
- buck install ReactAndroid/src/androidTest/buck-runner:instrumentation-tests --config build.threads=1
|
- buck install ReactAndroid/src/androidTest/buck-runner:instrumentation-tests --config build.threads=1
|
||||||
# run installed apk with tests
|
# run installed apk with tests
|
||||||
- source scripts/circle-ci-android-setup.sh && retry3 ./scripts/run-android-instrumentation-tests.sh com.facebook.react.tests
|
- node ./scripts/run-ci-android-instrumentation-tests.js --retries 3 --path ./ReactAndroid/src/androidTest/java/com/facebook/react/tests --package com.facebook.react.tests
|
||||||
|
|
||||||
- ./gradlew :ReactAndroid:testDebugUnitTest
|
- ./gradlew :ReactAndroid:testDebugUnitTest
|
||||||
# Deprecated: these tests are executed using Buck above, while we support Gradle we just make sure the test code compiles
|
# Deprecated: these tests are executed using Buck above, while we support Gradle we just make sure the test code compiles
|
||||||
|
|
|
@ -1,12 +1,16 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Python script to run instrumentation tests, copied from https://github.com/circleci/circle-dummy-android
|
||||||
|
# Example: ./scripts/run-android-instrumentation-tests.sh com.facebook.react.tests com.facebook.react.tests.ReactPickerTestCase
|
||||||
|
#
|
||||||
export PATH="$ANDROID_HOME/platform-tools:$ANDROID_HOME/tools:$PATH"
|
export PATH="$ANDROID_HOME/platform-tools:$ANDROID_HOME/tools:$PATH"
|
||||||
|
|
||||||
# clear the logs
|
# clear the logs
|
||||||
adb logcat -c
|
adb logcat -c
|
||||||
|
|
||||||
# run tests and check output
|
# run tests and check output
|
||||||
python - $1 << END
|
python - $1 $2 << END
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import subprocess as sp
|
import subprocess as sp
|
||||||
import sys
|
import sys
|
||||||
|
@ -14,8 +18,13 @@ import threading
|
||||||
import time
|
import time
|
||||||
|
|
||||||
done = False
|
done = False
|
||||||
test_app = sys.argv[1]
|
|
||||||
|
|
||||||
|
test_app = sys.argv[1]
|
||||||
|
test_class = None
|
||||||
|
|
||||||
|
if len(sys.argv) > 2:
|
||||||
|
test_class = sys.argv[2]
|
||||||
|
|
||||||
def update():
|
def update():
|
||||||
# prevent CircleCI from killing the process for inactivity
|
# prevent CircleCI from killing the process for inactivity
|
||||||
while not done:
|
while not done:
|
||||||
|
@ -28,8 +37,12 @@ t.start()
|
||||||
|
|
||||||
def run():
|
def run():
|
||||||
sp.Popen(['adb', 'wait-for-device']).communicate()
|
sp.Popen(['adb', 'wait-for-device']).communicate()
|
||||||
p = sp.Popen('adb shell am instrument -w %s/android.support.test.runner.AndroidJUnitRunner' % test_app,
|
if (test_class != None):
|
||||||
shell=True, stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
|
p = sp.Popen('adb shell am instrument -w -e class %s %s/android.support.test.runner.AndroidJUnitRunner'
|
||||||
|
% (test_class, test_app), shell=True, stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
|
||||||
|
else :
|
||||||
|
p = sp.Popen('adb shell am instrument -w %s/android.support.test.runner.AndroidJUnitRunner'
|
||||||
|
% (test_app), shell=True, stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
|
||||||
return p.communicate()
|
return p.communicate()
|
||||||
|
|
||||||
success = re.compile(r'OK \(\d+ tests\)')
|
success = re.compile(r'OK \(\d+ tests\)')
|
||||||
|
|
|
@ -0,0 +1,56 @@
|
||||||
|
/**
|
||||||
|
* Copyright (c) 2015-present, Facebook, Inc.
|
||||||
|
* All rights reserved.
|
||||||
|
*
|
||||||
|
* This source code is licensed under the BSD-style license found in the
|
||||||
|
* LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
* of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
*/
|
||||||
|
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This script runs instrumentation tests one by one with retries
|
||||||
|
* Instrumentation tests tend to be flaky, so rerunning them individually increases
|
||||||
|
* chances for success and reduces total average execution time.
|
||||||
|
*
|
||||||
|
* We assume that all instrumentation tests are flat in one folder
|
||||||
|
* Available arguments:
|
||||||
|
* --path - path to all .java files with tests
|
||||||
|
* --package - com.facebook.react.tests
|
||||||
|
* --retries [num] - how many times to retry possible flaky commands: npm install and running tests, default 1
|
||||||
|
*/
|
||||||
|
/*eslint-disable no-undef */
|
||||||
|
require('shelljs/global');
|
||||||
|
|
||||||
|
const argv = require('yargs').argv;
|
||||||
|
const numberOfRetries = argv.retries || 1;
|
||||||
|
const tryExecNTimes = require('./try-n-times');
|
||||||
|
const path = require('path');
|
||||||
|
|
||||||
|
// ReactAndroid/src/androidTest/java/com/facebook/react/tests/ReactHorizontalScrollViewTestCase.java
|
||||||
|
const testClasses = ls(`${argv.path}/*.java`)
|
||||||
|
.map(javaFile => {
|
||||||
|
// ReactHorizontalScrollViewTestCase
|
||||||
|
return path.basename(javaFile, '.java');
|
||||||
|
}).map(className => {
|
||||||
|
// com.facebook.react.tests.ReactHorizontalScrollViewTestCase
|
||||||
|
return argv.package + '.' + className;
|
||||||
|
});
|
||||||
|
|
||||||
|
let exitCode = 0;
|
||||||
|
testClasses.forEach((testClass) => {
|
||||||
|
if (tryExecNTimes(
|
||||||
|
() => {
|
||||||
|
exec('sleep 5s');
|
||||||
|
return exec(`./scripts/run-android-instrumentation-tests.sh ${argv.package} ${testClass}`).code;
|
||||||
|
},
|
||||||
|
numberOfRetries)) {
|
||||||
|
echo(`${testClass} failed ${numberOfRetries} times`);
|
||||||
|
exitCode = 1;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
exit(exitCode);
|
||||||
|
|
||||||
|
/*eslint-enable no-undef */
|
|
@ -27,6 +27,7 @@ const path = require('path');
|
||||||
|
|
||||||
const SCRIPTS = __dirname;
|
const SCRIPTS = __dirname;
|
||||||
const ROOT = path.normalize(path.join(__dirname, '..'));
|
const ROOT = path.normalize(path.join(__dirname, '..'));
|
||||||
|
const tryExecNTimes = require('./try-n-times');
|
||||||
|
|
||||||
const TEMP = exec('mktemp -d /tmp/react-native-XXXXXXXX').stdout.trim();
|
const TEMP = exec('mktemp -d /tmp/react-native-XXXXXXXX').stdout.trim();
|
||||||
// To make sure we actually installed the local version
|
// To make sure we actually installed the local version
|
||||||
|
@ -39,33 +40,6 @@ let SERVER_PID;
|
||||||
let APPIUM_PID;
|
let APPIUM_PID;
|
||||||
let exitCode;
|
let exitCode;
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Try executing a function n times recursively.
|
|
||||||
* Return 0 the first time it succeeds
|
|
||||||
* Return code of the last failed commands if not more retries left
|
|
||||||
* @funcToRetry - function that gets retried
|
|
||||||
* @retriesLeft - number of retries to execute funcToRetry
|
|
||||||
* @onEveryError - func to execute if funcToRetry returns non 0
|
|
||||||
*/
|
|
||||||
function tryExecNTimes(funcToRetry, retriesLeft, onEveryError) {
|
|
||||||
const exitCode = funcToRetry();
|
|
||||||
if (exitCode === 0) {
|
|
||||||
return exitCode;
|
|
||||||
} else {
|
|
||||||
if (onEveryError) {
|
|
||||||
onEveryError();
|
|
||||||
}
|
|
||||||
retriesLeft--;
|
|
||||||
echo(`Command failed, ${retriesLeft} retries left`);
|
|
||||||
if (retriesLeft === 0) {
|
|
||||||
return exitCode;
|
|
||||||
} else {
|
|
||||||
return tryExecNTimes(funcToRetry, retriesLeft, onEveryError);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// install CLI
|
// install CLI
|
||||||
cd('react-native-cli');
|
cd('react-native-cli');
|
||||||
|
|
|
@ -0,0 +1,27 @@
|
||||||
|
/**
|
||||||
|
* Try executing a function n times recursively.
|
||||||
|
* Return 0 the first time it succeeds
|
||||||
|
* Return code of the last failed commands if not more retries left
|
||||||
|
* @funcToRetry - function that gets retried
|
||||||
|
* @retriesLeft - number of retries to execute funcToRetry
|
||||||
|
* @onEveryError - func to execute if funcToRetry returns non 0
|
||||||
|
*/
|
||||||
|
function tryExecNTimes(funcToRetry, retriesLeft, onEveryError) {
|
||||||
|
const exitCode = funcToRetry();
|
||||||
|
if (exitCode === 0) {
|
||||||
|
return exitCode;
|
||||||
|
} else {
|
||||||
|
if (onEveryError) {
|
||||||
|
onEveryError();
|
||||||
|
}
|
||||||
|
retriesLeft--;
|
||||||
|
echo(`Command failed, ${retriesLeft} retries left`);
|
||||||
|
if (retriesLeft === 0) {
|
||||||
|
return exitCode;
|
||||||
|
} else {
|
||||||
|
return tryExecNTimes(funcToRetry, retriesLeft, onEveryError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = tryExecNTimes;
|
Загрузка…
Ссылка в новой задаче