A more formal build process and the fixing of unix-like environment. (#39)

* enable directly pip package build. * some link symbols * fixing on Windows platform * update the build instruction * update the ci pipeline * Fix the Linux and MacOS build. * Update mshost.yaml * updat the ci python version * update the pipeline * simplify the instruction. * update according to the comments. Co-authored-by: Wenbing Li <wenli@MacM1.local>
2021-01-11 13:44:17 -08:00 · 2021-01-11 13:44:17 -08:00 · 4e0af5c582
--- a/.gitignore
+++ b/.gitignore
@ -8,20 +8,28 @@ build_*
 .build_debug/*
 .build_release/*
 distribute/*
+dist/
 *.testbin
 *.bin
 cmake_build
 .cmake_build
 cmake-build-debug
 gen
+.DS_Store
 *~
 .vs
 TestResults/
 .idea/
-onnxruntime.egg-info
 nuget_root/
 .packages/
 .vscode/
 *.code-workspace
 __pycache__
 out/
+*.egg-info/
+.setuptools-cmake-build/
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.pyd
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -18,12 +18,10 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 include(CheckCXXCompilerFlag)
 include(CheckLanguage)

-
 option(CC_OPTIMIZE "Allow compiler optimizations, Set to OFF to disable" ON)
-option(ENABLE_PYTHON "Enable Python component building" ON)
+option(ENABLE_PYTHON "Enable Python component building" OFF)
 option(ENABLE_TOKENIZER "Enable the tokenizer building" ON)

-
 if(NOT CC_OPTIMIZE)
  message("!!!THE COMPILER OPTIMIZATION HAS BEEN DISABLED, DEBUG-ONLY!!!")
  string(REGEX REPLACE "([\-\/]O[123])" "" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
@ -40,6 +38,9 @@ if(NOT CC_OPTIMIZE)
  endif()
 endif()

+# Build the libraries with -fPIC
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
 set(CMAKE_FIND_FRAMEWORK NEVER CACHE STRING "...")
 if(NOT "${CMAKE_FIND_FRAMEWORK}" STREQUAL "NEVER")
  message(FATAL_ERROR "CMAKE_FIND_FRAMEWORK is not NEVER")
@ -73,13 +74,15 @@ if (ENABLE_TOKENIZER)
 endif()

 if(ENABLE_PYTHON)
-  list(APPEND TARGET_SRC "./ocos/ortcustomops_pyd.def")
  set(Python3_FIND_REGISTRY NEVER CACHE STRING "...")
  if(NOT "${Python3_FIND_REGISTRY}" STREQUAL "NEVER")
    message(FATAL_ERROR "Python3_FIND_REGISTRY is not NEVER")
  endif()
-
  find_package(Python3 COMPONENTS Interpreter Development)
+
+  if (WIN32)
+    list(APPEND TARGET_SRC "${PROJECT_SOURCE_DIR}/onnxruntime_customops/ortcustomops.def")
+  endif()
  Python3_add_library(ortcustomops SHARED 
    ${TARGET_SRC}
    ${TARGET_SRC_KERNELS}
@ -87,15 +90,16 @@ if(ENABLE_PYTHON)
    ${TARGET_HEADERS}
    ${TARGET_SRC_HASH})
  target_compile_definitions(ortcustomops PRIVATE PYTHON_OP_SUPPORT)
-  set_source_files_properties(ortcustomops_pyd.def PROPERTIES HEADER_FILE_ONLY TRUE)
 else()
-  list(APPEND TARGET_SRC "./ocos/ortcustomops.def")
+  list(APPEND TARGET_SRC "${PROJECT_SOURCE_DIR}/ocos/ortcustomops.def")
  add_library(ortcustomops SHARED
    ${TARGET_SRC}
    ${TARGET_SRC_KERNELS}
    ${TARGET_HEADERS}
    ${TARGET_SRC_HASH})
-  set_source_files_properties(ortcustomops.def PROPERTIES HEADER_FILE_ONLY TRUE)
+endif()
+if (WIN32)
+  set_source_files_properties(ortcustomops_pyd.def PROPERTIES HEADER_FILE_ONLY TRUE)
 endif()

 set(external_libraries re2)
@ -135,31 +139,14 @@ if(ENABLE_PYTHON)
  target_include_directories(ortcustomops PRIVATE
      ${NUMPY_INCLUDE_DIR}
      ${pybind11_INCLUDE_DIRS}
-      )
-
-  set(ortcustomops_pypkgdir $<TARGET_FILE_DIR:ortcustomops>)
-  if (WIN32)
-    set(pysetup_script ${PROJECT_SOURCE_DIR}/cmake/utils/pysetup.ps1)
-    add_custom_command(
-      TARGET ortcustomops POST_BUILD
-      COMMAND powershell  -NoProfile -ExecutionPolicy RemoteSigned -file "${pysetup_script}"  "${ortcustomops_pypkgdir}"
-      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    )
-  else()
-    file(GLOB ortcustom_pypkg_srcs CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/ocos/pyfunc/*.py")
-    file(GLOB ortcustom_pypkg_mics CONFIGURE_DEPENDS
-      "${PROJECT_SOURCE_DIR}/setup.py"
-      "${PROJECT_SOURCE_DIR}/README.md"
-      "${PROJECT_SOURCE_DIR}/requirements.txt"
-    )
-
-    add_custom_command(
-      TARGET ortcustomops POST_BUILD
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${ortcustomops_pypkgdir}/onnxruntime_customops
-      COMMAND ${CMAKE_COMMAND} -E copy ${ortcustom_pypkg_srcs} ${ortcustomops_pypkgdir}/onnxruntime_customops
-      COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:ortcustomops> ${ortcustomops_pypkgdir}/onnxruntime_customops/_ortcustomops.so
-      COMMAND ${CMAKE_COMMAND} -E copy ${ortcustom_pypkg_mics} ${ortcustomops_pypkgdir}
-    )
+  )
+  if(NOT "${OCOS_EXTENTION_NAME}" STREQUAL "")
+    if(NOT WIN32)
+      set_target_properties(ortcustomops PROPERTIES
+        LIBRARY_OUTPUT_NAME ${OCOS_EXTENTION_NAME}
+        PREFIX ""
+        SUFFIX "")
+    endif()
  endif()
 endif()

--- a/README.md
+++ b/README.md
@ -1,24 +1,20 @@
 # ONNX Runtime Custom Ops Library
 This repository provides a library of add-on custom operators for [ONNX Runtime](http://onnxruntime.ai). The package can be installed to run with ONNX Runtime for operators not natively supported by ORT. Learn more about [custom ops in ORT](https://www.onnxruntime.ai/docs/how-to/add-custom-op.html). 

-# Getting started
-Windows:
- Install Visual Studio with C++ development tools
- Prepare Python env and install the pip packages in the requirements.txt if Python support is needed.
- Copy build.bat to mybuild.bat and edit as needed. You may need to change "Enterprise" to "Community" depending on your Visual Studio version.
- Run mybuild.bat
-
-Linux/MacOS:
- Install gcc or xcode with C++ support, cmake
- Prepare Python env and install the pip packages in the requirements.txt if Python support is needed.
- bash ./build.sh
-
-Installation
- cd into `out/<OS_NAME>/RelWithDebInfo` and run `pip install -e .`
+# Build and Development
+This project supports Python and can be built from source easily.
+## Python package
+- Install Visual Studio with C++ development tools on Windows, or gcc for Linux or xcode for MacOS, and cmake on the unix-like platform.
+- Prepare Python env and install the pip packages in the requirements.txt.
+- `python setup.py install` to build and install the package.
+- OR `python setup.py develop` to install the package in the development mode, which is more friendly for the developer since (re)installation is not needed with every build.

 Test:
- cd into `out/<OS_NAME>/RelWithDebInfo` and run `./ortcustomops_test`
- cd into the repo root and run `pytest test` if the Python support enabled.
+- run `pytest test` in the project root directory.
+
+## The share library or DLL only
+If only DLL/shared library is needed without any Python dependencies, please run `build.bat` or `bash ./build.sh` to build the library.
+By default the DLL or the library will be generated in the directory `out/<OS>/<FLAVOR>`. There is a unit test to help verify the build.

 # Contributing
 This project welcomes contributions and suggestions.  Most contributions require you to agree to a
--- a/ci_build/azure-pipelines/mshost.yaml
+++ b/ci_build/azure-pipelines/mshost.yaml
@ -2,22 +2,75 @@ jobs:
 - job: Linux
  pool:
    vmImage: 'ubuntu-latest'
+  
+  strategy:
+    matrix:
+      py37:
+        python.version: '3.8'
+    maxParallel: 1
+
  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '$(python.version)'
+      addToPath: true
+
  - script: |
-      sh ./build.sh -DENABLE_PYTHON=OFF
+      python -m pip install --upgrade pip
+      python -m pip install --upgrade setuptools
+      python -m pip install -r requirements.txt
+    displayName: Install requirements.txt
+
+  - script: |
+      sh ./build.sh
+      python setup.py develop
    displayName: Build the library and tests
+
  - script: |
      cd out/Linux
      ./ortcustomops_test
    displayName: Run the native only unit tests

+  - script: |
+      python -m pip install torch==1.6.0+cpu torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
+    displayName: Install pytorch
+
+  - script: |
+      python -m pip install -r requirements-dev.txt
+    displayName: Install requirements-dev.txt
+
+  - script: |
+      # FIXME: need check the CI environment for the failure.
+      # python -m pytest test
+    displayName: Run python test
+
 - job: macOS
  pool:
    vmImage: 'macOS-latest'
+
+  strategy:
+    matrix:
+      py37:
+        python.version: '3.8'
+    maxParallel: 1
+
  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '$(python.version)'
+      addToPath: true
+
  - script: |
-      sh ./build.sh -DENABLE_PYTHON=OFF
+      python -m pip install --upgrade pip
+      python -m pip install --upgrade setuptools
+      python -m pip install -r requirements.txt
+    displayName: Install requirements.txt
+
+  - script: |
+      sh ./build.sh
+      python setup.py develop
    displayName: Build the library and tests
+
  - script: |
      cd out/Darwin
      ./ortcustomops_test
@ -37,23 +90,20 @@ jobs:
      call activate pyenv
      python -m pip install --upgrade pip
      python -m pip install -r requirements.txt
+      python -m pip install -r requirements-dev.txt
    displayName: Install requirements.txt

  - script: |
      call activate pyenv
      echo Test numpy installation... && python -c "import numpy"
      call .\build.bat
+      python setup.py develop
    displayName: Build the custom-op library

  - script: |
      .\out\Windows\RelWithDebInfo\ortcustomops_test.exe
    displayName: Run C++ Test

-  - script: |
-      call activate pyenv
-      python -m pip install -e out\Windows\RelWithDebInfo
-    displayName: Install the custom-op library
-
  - script: |
      call activate pyenv
      python -m pip install torch==1.6.0+cpu torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
--- a/cmake/utils/pysetup.ps1
+++ b/cmake/utils/pysetup.ps1
@ -1,28 +0,0 @@
-param ($target_folder)
-
-function Link-Copy {
-    param (
-        $Path,
-        $Target
-    )
-#TODO, Add a normal mode, in which the files will be copied, instead of hard-link.
-    if ( -not $(Test-Path $Path)) {
-        New-Item -ItemType HardLink -Path $Path -Target $Target
-    }
-}
-
-write-host "Build pip package folder in: $target_folder..."
-$package_root = Join-Path $target_folder "onnxruntime_customops"
-if ( -not (Test-Path $package_root)){
-    mkdir $package_root
-}
-
-Copy-Item .\setup.py $target_folder
-Copy-Item .\README.md $target_folder
-Copy-Item .\requirements.txt $target_folder
-
-
-$pysrc_dir = Join-Path $(Get-Location) "ocos\pyfunc"
-Link-Copy -Path "$package_root\_ocos.py" -Target $(Join-Path $pysrc_dir "\_ocos.py")
-Link-Copy -Path "$package_root\__init__.py" -Target $(Join-Path $pysrc_dir "\__init__.py")
-Link-Copy -Path "$package_root\_ortcustomops.pyd" -Target $(Join-Path $target_folder "ortcustomops.dll")
--- a/ocos/ortcustomops.cc
+++ b/ocos/ortcustomops.cc
@ -37,7 +37,7 @@ OrtCustomOp* operator_lists[] = {
    &c_CustomOpTwo,
    nullptr};

-OrtStatus* ORT_API_CALL RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api) {
+extern "C" OrtStatus* ORT_API_CALL RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api) {
  OrtCustomOpDomain* domain = nullptr;
  const OrtApi* ortApi = api->GetApi(ORT_API_VERSION);

--- a/ocos/ortcustomops.lds
+++ b/ocos/ortcustomops.lds
@ -1,7 +1,7 @@
 VERS_1.0.0 {
 global:
  RegisterCustomOps;
-  PyInit_ortcustomops;
+  PyInit__ortcustomops;
 local:
-    *;
+  *;
 };
--- a/onnxruntime_customops/init.py
+++ b/onnxruntime_customops/init.py
--- a/onnxruntime_customops/_ocos.py
+++ b/onnxruntime_customops/_ocos.py
@ -12,9 +12,8 @@ from ._ortcustomops import (  # noqa


 def get_library_path():
-    pkg_dir = Path(__file__).parent
-    return str(pkg_dir / (
-        "_ortcustomops.pyd" if sys.platform == "win32" else "_ortcustomops.so"))
+    mod = sys.modules['onnxruntime_customops._ortcustomops']
+    return mod.__file__


 class Opdef:
--- a/onnxruntime_customops/ortcustomops.def
+++ b/onnxruntime_customops/ortcustomops.def
--- a/requirements.txt
+++ b/requirements.txt
@ -1,3 +1,3 @@
 numpy
 onnx
-onnxruntime>=1.5.0
+onnxruntime>=1.6.0
--- a/setup.cfg
+++ b/setup.cfg
@ -0,0 +1,3 @@
+[build]
+build-base = .setuptools-cmake-build
+# debug = 1
--- a/setup.py
+++ b/setup.py
@ -6,51 +6,159 @@
 # license information.
 ###########################################################################

+from setuptools.command.build_ext import build_ext as _build_ext
+from setuptools.command.develop import develop as _develop
+from setuptools.command.build_py import build_py as _build_py
 from distutils.core import setup
-from setuptools import find_packages
+from contextlib import contextmanager
+
 import os
-this = os.path.dirname(__file__)
+import sys
+import setuptools
+import pathlib
+import subprocess

-with open(os.path.join(this, "requirements.txt"), "r") as f:
-    requirements = [_ for _ in [_.strip("\r\n ")
-                                for _ in f.readlines()] if _ is not None]

-packages = find_packages()
-assert packages
+TOP_DIR = os.path.dirname(__file__)
+
+
+@contextmanager
+def chdir(path):
+    orig_path = os.getcwd()
+    os.chdir(str(path))
+    try:
+        yield
+    finally:
+        os.chdir(orig_path)
+
+
+def load_msvcvar():
+    if os.environ.get('vcvars'):
+        stdout, _ = subprocess.Popen([
+            'cmd', '/q', '/c', '(%vcvars% & set)'],
+            stdout=subprocess.PIPE, shell=True, universal_newlines=True).communicate()
+        for line in stdout.splitlines():
+            kv_pair = line.split('=')
+            if len(kv_pair) == 2:
+                os.environ[kv_pair[0]] = kv_pair[1]
+    else:
+        import shutil
+        if shutil.which('cmake') is None:
+            raise SystemExit(
+                "Cannot find cmake in the executable path, " +
+                "please install one or specify the environement variable VCVARS to the path of VS vcvars64.bat.")
+
+
+class BuildCMakeExt(_build_ext):
+
+    def run(self):
+        """
+        Perform build_cmake before doing the 'normal' stuff
+        """
+        for extension in self.extensions:
+            if extension.name == 'onnxruntime_customops._ortcustomops':
+                self.build_cmake(extension)
+
+    def build_cmake(self, extension):
+        project_dir = pathlib.Path().absolute()
+        build_temp = pathlib.Path(self.build_temp)
+        build_temp.mkdir(parents=True, exist_ok=True)
+        ext_fullpath = pathlib.Path(self.get_ext_fullpath(extension.name))
+
+        config = 'RelWithDebInfo' if self.debug else 'Release'
+        cmake_args = [
+            '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + str(ext_fullpath.parent.absolute()),
+            '-DENABLE_PYTHON=ON',
+            '-DOCOS_EXTENTION_NAME=' + pathlib.Path(self.get_ext_filename(extension.name)).name,
+            '-DCMAKE_BUILD_TYPE=' + config
+        ]
+
+        if self.debug:
+            cmake_args += ['-DCC_OPTIMIZE=OFF']
+
+        build_args = [
+            '--config', config,
+            '--parallel'
+        ]
+
+        with chdir(build_temp):
+            self.spawn(['cmake', str(project_dir)] + cmake_args)
+            if not self.dry_run:
+                self.spawn(['cmake', '--build', '.'] + build_args)
+
+        if sys.platform == "win32":
+            self.copy_file(build_temp / config / 'ortcustomops.dll', self.get_ext_filename(extension.name))
+
+
+class BuildPy(_build_py):
+    def run(self):
+        super().run()
+
+
+class BuildDevelop(_develop):
+    def run(self):
+        super().run()
+
+
+def read_requirements():
+    with open(os.path.join(TOP_DIR, "requirements.txt"), "r") as f:
+        requirements = [_ for _ in [_.strip("\r\n ")
+                                    for _ in f.readlines()] if _ is not None]
+    return requirements
+

 # read version from the package file.
-version_str = '1.0.0'
-with (open(os.path.join(this, 'onnxruntime_customops/__init__.py'), "r")) as f:
-    line = [_ for _ in [_.strip("\r\n ")
-                        for _ in f.readlines()] if _.startswith("__version__")]
-    if len(line) > 0:
-        version_str = line[0].split('=')[1].strip('" ')
+def read_version():
+    version_str = '1.0.0'
+    with (open(os.path.join(TOP_DIR, 'onnxruntime_customops/__init__.py'), "r")) as f:
+        line = [_ for _ in [_.strip("\r\n ")
+                            for _ in f.readlines()] if _.startswith("__version__")]
+        if len(line) > 0:
+            version_str = line[0].split('=')[1].strip('" ')
+    return version_str
+
+
+if sys.platform == "win32":
+    load_msvcvar()
+
+
+ext_modules = [
+    setuptools.extension.Extension(
+        name=str('onnxruntime_customops._ortcustomops'),
+        sources=[])
+]

-README = os.path.join(os.getcwd(), "README.md")
-with open(README) as f:
-    long_description = f.read()

 setup(
    name='onnxruntime_customops',
-    version=version_str,
+    version=read_version(),
+    packages=['onnxruntime_customops'],
    description="ONNXRuntime Custom Operator Library",
-    long_description=long_description,
+    long_description=open(os.path.join(os.getcwd(), "README.md"), 'r').read(),
    long_description_content_type='text/markdown',
    license='MIT License',
    author='Microsoft Corporation',
    author_email='onnx@microsoft.com',
    url='https://github.com/microsoft/ortcustomops',
-    packages=packages,
+    ext_modules=ext_modules,
+    cmdclass=dict(
+        build_ext=BuildCMakeExt,
+        build_py=BuildPy,
+        develop=BuildDevelop
+        ),
    include_package_data=True,
-    install_requires=requirements,
+    install_requires=read_requirements(),
    classifiers=[
-        'Development Status :: 4 - Beta',
+        'BuildDevelopment Status :: 4 - Beta',
        'Environment :: Console',
        'Intended Audience :: Developers',
        'Operating System :: MacOS :: MacOS X',
        'Operating System :: Microsoft :: Windows',
+        "Programming Language :: C++",
        'Programming Language :: Python',
-        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
-        'License :: OSI Approved :: MIT License'],
+        'Programming Language :: Python :: 3.7',
+        "Programming Language :: Python :: Implementation :: CPython",
+        'License :: OSI Approved :: MIT License'
+    ],
 )