Create nightly build for python packages (#817)

* Enable nightly build * Update fetch file names * Fix * Update setup.py * Update run_dockerbuild.sh * Resolve comments
2019-04-11 22:06:18 -07:00 · 2019-04-11 22:06:18 -07:00 · 1936d141a7
--- a/setup.py
+++ b/setup.py
@ -7,14 +7,29 @@ from setuptools import setup, find_packages
 from os import path, getcwd
 import platform
 import sys
+import datetime

+nightly_build = False
 package_name = 'onnxruntime'
 if '--use_tensorrt' in sys.argv:
    package_name = 'onnxruntime-gpu-tensorrt'
    sys.argv.remove('--use_tensorrt')
+    if '--nightly_build' in sys.argv:
+        package_name = 'ort-trt-nightly'
+        nightly_build = True
+        sys.argv.remove('--nightly_build')
 elif '--use_cuda' in sys.argv:
    package_name = 'onnxruntime-gpu'
    sys.argv.remove('--use_cuda')
+    if '--nightly_build' in sys.argv:
+        package_name = 'ort-gpu-nightly'
+        nightly_build = True
+        sys.argv.remove('--nightly_build')
+
+if '--nightly_build' in sys.argv:
+    package_name = 'ort-nightly'
+    nightly_build = True
+    sys.argv.remove('--nightly_build')

 try:
    from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
@ -52,9 +67,13 @@ if not path.exists(README):
 with open(README) as f:
    long_description = f.read()

+
 version_number = ''
 with open('VERSION_NUMBER') as f:
    version_number = f.readline().strip()
+if nightly_build:
+    date_suffix = str(datetime.datetime.now().date().strftime("%m%d"))
+    version_number = version_number + ".dev" + date_suffix

 # Setup
 setup(
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@ -135,6 +135,7 @@ Use the individual flags to only run the specified stages.
    parser.add_argument("--tensorrt_home", help="Path to TensorRT installation dir")
    parser.add_argument("--use_full_protobuf", action='store_true', help="Use the full protobuf library")
    parser.add_argument("--disable_contrib_ops", action='store_true', help="Disable contrib ops (reduces binary size)")
+    parser.add_argument("--skip_onnx_tests", action='store_true', help="Explicitly disable all onnx related tests")
    return parser.parse_args()

 def resolve_executable_path(command_or_path):
@ -556,17 +557,26 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_para
        else:
            run_subprocess([exe] + cmd, cwd=cwd)

-def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_tensorrt):
+def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_tensorrt, nightly_build = False):
    for config in configs:
        cwd = get_config_build_dir(build_dir, config)
+
        if is_windows():
            cwd = os.path.join(cwd, config)
-        if use_tensorrt:
-            run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_tensorrt'], cwd=cwd)
-        elif use_cuda:
-            run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_cuda'], cwd=cwd)
+        if nightly_build:
+            if use_tensorrt:
+                run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_tensorrt', '--nightly_build'], cwd=cwd)
+            elif use_cuda:
+                run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_cuda', '--nightly_build'], cwd=cwd)
+            else:
+                run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--nightly_build'], cwd=cwd)
        else:
-            run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel'], cwd=cwd)
+            if use_tensorrt:
+                run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_tensorrt'], cwd=cwd)
+            elif use_cuda:
+                run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_cuda'], cwd=cwd)
+            else:
+                run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel'], cwd=cwd)
        if is_ubuntu_1604():
            run_subprocess([os.path.join(source_dir, 'rename_manylinux.sh')], cwd=cwd+'/dist')

@ -716,10 +726,12 @@ def main():
    if (args.build):
        build_targets(cmake_path, build_dir, configs, args.parallel)

-    if (args.test):
-        run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, args.enable_pybind, args.use_tvm, args.use_tensorrt)
+    if args.test :
+        run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs,
+                              args.enable_pybind if not args.skip_onnx_tests else False,
+                              args.use_tvm, args.use_tensorrt)
        # run the onnx model tests if requested explicitly.
-        if (args.enable_onnx_tests):
+        if args.enable_onnx_tests and not args.skip_onnx_tests:
            # directory from ONNX submodule with ONNX test data
            onnx_test_data_dir = '/data/onnx'
            if is_windows() or not os.path.exists(onnx_test_data_dir):
@ -741,7 +753,8 @@ def main():

    if args.build:
        if args.build_wheel:
-            build_python_wheel(source_dir, build_dir, configs, args.use_cuda, args.use_tensorrt)
+            nightly_build = bool(os.getenv('NIGHTLY_BUILD') == '1')
+            build_python_wheel(source_dir, build_dir, configs, args.use_cuda, args.use_tensorrt, nightly_build)
    
    if args.gen_doc:
        generate_documentation(source_dir, build_dir, configs)
--- a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
+++ b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
@ -22,7 +22,7 @@ jobs:
      displayName: 'Copy Python Wheel to:  $(Build.ArtifactStagingDirectory)'
      inputs:
        SourceFolder: '$(Build.BinariesDirectory)'
-        Contents: 'Release/dist/onnxruntime-*-manylinux1_x86_64.whl'
+        Contents: 'Release/dist/*-manylinux1_x86_64.whl'
        TargetFolder: '$(Build.ArtifactStagingDirectory)'

    - task: PublishBuildArtifacts@1
@ -59,7 +59,7 @@ jobs:
      displayName: 'Copy Python Wheel to:  $(Build.ArtifactStagingDirectory)'
      inputs:
        SourceFolder: '$(Build.BinariesDirectory)'
-        Contents: 'Release/dist/onnxruntime*-manylinux1_x86_64.whl'
+        Contents: 'Release/dist/*-manylinux1_x86_64.whl'
        TargetFolder: '$(Build.ArtifactStagingDirectory)'

    - task: PublishBuildArtifacts@1
@ -104,7 +104,7 @@ jobs:
      displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
      inputs:
        SourceFolder: '$(buildDirectory)'
-        Contents: '**\dist\onnxruntime-*.whl'
+        Contents: '**\dist\*.whl'
        TargetFolder: '$(Build.ArtifactStagingDirectory)'

    - task: PublishBuildArtifacts@1
@ -171,7 +171,7 @@ jobs:
      displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
      inputs:
        SourceFolder: '$(buildDirectory)'
-        Contents: '**\dist\onnxruntime_gpu-*.whl'
+        Contents: '**\dist\*.whl'
        TargetFolder: '$(Build.ArtifactStagingDirectory)'

    - task: PublishBuildArtifacts@1
@ -221,7 +221,7 @@ jobs:
      displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
      inputs:
        SourceFolder: '$(Build.SourcesDirectory)'
-        Contents: '**/dist/onnxruntime-*.whl'
+        Contents: '**/dist/*.whl'
        TargetFolder: '$(Build.ArtifactStagingDirectory)'

    - task: PublishBuildArtifacts@1
--- a/tools/ci_build/github/linux/run_dockerbuild.sh
+++ b/tools/ci_build/github/linux/run_dockerbuild.sh
@ -51,6 +51,11 @@ fi
 set +e
 mkdir -p ~/.cache/onnxruntime
 mkdir -p ~/.onnx
+
+if [ -z "$NIGHTLY_BUILD" ]; then
+set NIGHTLY_BUILD=0
+fi
+
 if [ $BUILD_DEVICE = "cpu" ]; then
    docker rm -f "onnxruntime-$BUILD_DEVICE" || true
    docker run -h $HOSTNAME \
@ -59,6 +64,7 @@ if [ $BUILD_DEVICE = "cpu" ]; then
        --volume "$BUILD_DIR:/build" \
        --volume "$HOME/.cache/onnxruntime:/home/onnxruntimedev/.cache/onnxruntime" \
        --volume "$HOME/.onnx:/home/onnxruntimedev/.onnx" \
+        -e NIGHTLY_BUILD \
        "onnxruntime-$IMAGE" \
        /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \
         -d $BUILD_DEVICE -x "$BUILD_EXTR_PAR" &
@ -71,6 +77,7 @@ else
        --volume "$BUILD_DIR:/build" \
        --volume "$HOME/.cache/onnxruntime:/home/onnxruntimedev/.cache/onnxruntime" \
        --volume "$HOME/.onnx:/home/onnxruntimedev/.onnx" \
+        -e NIGHTLY_BUILD \
        "onnxruntime-$IMAGE" \
        /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \
        -d $BUILD_DEVICE -x "$BUILD_EXTR_PAR" &