Add UT for Azure Ops during packaging (#502)
* set before-test * test cmd * clean in yml * restore toml * add ut for triton endpoints * reset working path * rename suffix * install ort * pip install * make env * add extra env * make executable * set dir for linux * add switch * set env default * skip tests * simplify env * clean env for official --------- Co-authored-by: Randy Shuai <rashuai@microsoft.com> Co-authored-by: Wenbing Li <10278425+wenbingl@users.noreply.github.com>
This commit is contained in:
RandySheriffH
GitHub
Родитель
922b7cc387
Коммит
9e7f8e5b1d
|
|
@ -16,13 +16,10 @@ parameters: # parameters are shown up in ADO UI in a build queue time
|
|||
type: boolean
|
||||
default: false
|
||||
|
||||
- name: BuildAzureOp
|
||||
displayName: 'Build and test AzureEP ops'
|
||||
- name: ExtraEnv
|
||||
displayName: 'Extra env variable set to CIBW_ENVIRONMENT'
|
||||
type: string
|
||||
values:
|
||||
- '0'
|
||||
- '1'
|
||||
default: '0'
|
||||
default: 'None=None'
|
||||
|
||||
variables:
|
||||
CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)]
|
||||
|
|
@ -35,7 +32,7 @@ variables:
|
|||
CIBW_BUILD: cp3{8,9,10,11}-*amd64
|
||||
CIBW_ARCHS: AMD64
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
CIBW_ENVIRONMENT: "OCOS_ENABLE_AZURE=${{ parameters.BuildAzureOp }}"
|
||||
CIBW_ENVIRONMENT: "${{ parameters.ExtraEnv }}"
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
|
|
@ -107,6 +104,7 @@ extends:
|
|||
python -m pip install --upgrade pip
|
||||
python -m pip install cibuildwheel numpy
|
||||
python -m cibuildwheel --platform windows --archs AMD64 --output-dir $(REPOROOT)\out
|
||||
tools\install_deps.bat uninstall
|
||||
displayName: Build wheels
|
||||
- task: SDLNativeRules@3
|
||||
inputs:
|
||||
|
|
|
|||
|
|
@ -16,13 +16,10 @@ parameters: # parameters are shown up in ADO UI in a build queue time
|
|||
type: boolean
|
||||
default: false
|
||||
|
||||
- name: BuildAzureOp
|
||||
displayName: 'Build and test AzureEP ops'
|
||||
- name: ExtraEnv
|
||||
displayName: 'Extra env variable set to CIBW_ENVIRONMENT'
|
||||
type: string
|
||||
values:
|
||||
- '0'
|
||||
- '1'
|
||||
default: '0'
|
||||
default: 'None=None'
|
||||
|
||||
variables:
|
||||
CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)]
|
||||
|
|
@ -36,7 +33,7 @@ variables:
|
|||
CIBW_ARCHS: AMD64
|
||||
CIBW_BUILD_VERBOSITY: 1
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
CIBW_ENVIRONMENT: "OCOS_ENABLE_AZURE=${{ parameters.BuildAzureOp }}"
|
||||
CIBW_ENVIRONMENT: "${{ parameters.ExtraEnv }}"
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
|
|
@ -108,6 +105,7 @@ extends:
|
|||
python -m pip install --upgrade pip
|
||||
python -m pip install cibuildwheel numpy
|
||||
python -m cibuildwheel --platform windows --archs AMD64 --output-dir $(REPOROOT)\out
|
||||
tools\install_deps.bat uninstall
|
||||
displayName: Build wheels
|
||||
- task: SDLNativeRules@3
|
||||
inputs:
|
||||
|
|
|
|||
|
|
@ -1,11 +1,8 @@
|
|||
parameters:
|
||||
- name: BuildAzureOp
|
||||
displayName: 'Build and test AzureEP ops'
|
||||
- name: ExtraEnv
|
||||
displayName: 'Extra env variable set to CIBW_ENVIRONMENT'
|
||||
type: string
|
||||
values:
|
||||
- '0'
|
||||
- '1'
|
||||
default: '0'
|
||||
default: 'None=None'
|
||||
|
||||
jobs:
|
||||
- job: linux
|
||||
|
|
@ -13,7 +10,7 @@ jobs:
|
|||
pool: {name: 'Azure-Pipelines-EO-Ubuntu-2004-aiinfra'}
|
||||
variables:
|
||||
CIBW_BUILD: "cp3{8,9,10,11}-*"
|
||||
CIBW_ENVIRONMENT: "OCOS_ENABLE_AZURE=${{ parameters.BuildAzureOp }}"
|
||||
CIBW_ENVIRONMENT: "${{ parameters.ExtraEnv }}"
|
||||
|
||||
steps:
|
||||
- task: UsePythonVersion@0
|
||||
|
|
|
|||
|
|
@ -379,12 +379,6 @@ if(OCOS_ENABLE_BLINGFIRE)
|
|||
endif()
|
||||
|
||||
if(OCOS_ENABLE_AZURE)
|
||||
|
||||
if ($ENV{TEST_AZURE_INVOKERS_AS_CPU_OP} MATCHES "ON")
|
||||
message(STATUS "Azure inovkers will be testable as cpu ops")
|
||||
add_compile_definitions(TEST_AZURE_INVOKERS_AS_CPU_OP)
|
||||
endif()
|
||||
|
||||
# Azure endpoint invokers
|
||||
include(triton)
|
||||
file(GLOB TARGET_SRC_AZURE "operators/azure/*.cc" "operators/azure/*.h*")
|
||||
|
|
|
|||
|
|
@ -374,7 +374,7 @@ struct Variadic : public TensorBase {
|
|||
tensor = std::make_unique<Custom::Tensor<int64_t>>(api, ctx, ith_input, true);
|
||||
break;
|
||||
case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING:
|
||||
tensor = std::make_unique<Custom::Tensor<std::string_view>>(api, ctx, ith_input, true);
|
||||
tensor = std::make_unique<Custom::Tensor<std::string>>(api, ctx, ith_input, true);
|
||||
break;
|
||||
default:
|
||||
ORTX_CXX_API_THROW("unknow input type", ORT_RUNTIME_EXCEPTION);
|
||||
|
|
|
|||
|
|
@ -343,11 +343,15 @@ void AzureTritonInvoker::Compute(const ortc::Variadic& inputs, ortc::Variadic& o
|
|||
triton_input_vec.emplace_back(triton_input);
|
||||
|
||||
triton_inputs.push_back(triton_input);
|
||||
// todo - test string
|
||||
const float* data_raw = reinterpret_cast<const float*>(inputs[ith_input]->DataRaw());
|
||||
size_t size_in_bytes = inputs[ith_input]->SizeInBytes();
|
||||
err = triton_input->AppendRaw(reinterpret_cast<const uint8_t*>(data_raw), size_in_bytes);
|
||||
CHECK_TRITON_ERR(err, "failed to append raw data to input");
|
||||
if ("BYTES" == triton_data_type) {
|
||||
const auto* string_tensor = reinterpret_cast<const ortc::Tensor<std::string>*>(inputs[ith_input].get());
|
||||
triton_input->AppendFromString(string_tensor->Data());
|
||||
} else {
|
||||
const float* data_raw = reinterpret_cast<const float*>(inputs[ith_input]->DataRaw());
|
||||
size_t size_in_bytes = inputs[ith_input]->SizeInBytes();
|
||||
err = triton_input->AppendRaw(reinterpret_cast<const uint8_t*>(data_raw), size_in_bytes);
|
||||
CHECK_TRITON_ERR(err, "failed to append raw data to input");
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t ith_output = 0; ith_output < output_names_.size(); ++ith_output) {
|
||||
|
|
@ -411,15 +415,11 @@ const std::vector<const OrtCustomOp*>& AzureInvokerLoader() {
|
|||
static OrtOpLoader op_loader(CustomAzureStruct("AzureAudioInvoker", AzureAudioInvoker),
|
||||
CustomAzureStruct("AzureTritonInvoker", AzureTritonInvoker),
|
||||
CustomAzureStruct("AzureAudioInvoker", AzureAudioInvoker),
|
||||
CustomAzureStruct("AzureTextInvoker", AzureTextInvoker)
|
||||
|
||||
#ifdef TEST_AZURE_INVOKERS_AS_CPU_OP
|
||||
,
|
||||
CustomAzureStruct("AzureTextInvoker", AzureTextInvoker),
|
||||
CustomCpuStruct("AzureAudioInvoker", AzureAudioInvoker),
|
||||
CustomCpuStruct("AzureTritonInvoker", AzureTritonInvoker),
|
||||
CustomCpuStruct("AzureAudioInvoker", AzureAudioInvoker),
|
||||
CustomCpuStruct("AzureTextInvoker", AzureTextInvoker)
|
||||
#endif
|
||||
);
|
||||
return op_loader.GetCustomOps();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ line-length = 120
|
|||
[[tool.cibuildwheel.overrides]]
|
||||
select = "*-manylinux*x86_64*"
|
||||
before-all = "./tools/install_deps.sh many64"
|
||||
test-command = "{project}/tools/test_cibuildwheel.sh {project}"
|
||||
|
||||
[[tool.cibuildwheel.overrides]]
|
||||
select = "*-manylinux*i686*"
|
||||
|
|
@ -23,4 +24,4 @@ before-all = "./tools/install_deps.sh musl"
|
|||
[[tool.cibuildwheel.overrides]]
|
||||
select = "*win*"
|
||||
before-build = "tools\\install_deps.bat install"
|
||||
after-build = "tools\\install_deps.bat uninstall"
|
||||
test-command = "{project}\\tools\\test_cibuildwheel.bat {project}"
|
||||
|
|
|
|||
Двоичный файл не отображается.
Двоичный файл не отображается.
|
|
@ -0,0 +1,17 @@
|
|||
:…
|
||||
™
|
||||
|
||||
auth_token
|
||||
text embeddingembedding_invoker"AzureTextInvoker*4
|
||||
model_uri"$https://api.openai.com/v1/embeddings *
|
||||
verbose :ai.onnx.contribgraphZ!
|
||||
|
||||
auth_token
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿZ
|
||||
text
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿb
|
||||
embedding
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿB
|
||||
Двоичный файл не отображается.
|
|
@ -0,0 +1,24 @@
|
|||
:Í
|
||||
Ò
|
||||
|
||||
auth_token
|
||||
X
|
||||
YZtriton_invoker"AzureTritonInvoker*
|
||||
|
||||
model_name"addf *G
|
||||
model_uri"7https://endpoint-5095584.westus2.inference.ml.azure.com *
|
||||
model_version"1 *
|
||||
verbose"1 :ai.onnx.contribgraphZ!
|
||||
|
||||
auth_token
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿZ
|
||||
X
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿZ
|
||||
Y
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿb
|
||||
Z
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿB
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
:Î
|
||||
Ó
|
||||
|
||||
auth_token
|
||||
X
|
||||
YZtriton_invoker"AzureTritonInvoker*
|
||||
|
||||
model_name"addf8 *G
|
||||
model_uri"7https://endpoint-3586177.westus2.inference.ml.azure.com *
|
||||
model_version"1 *
|
||||
verbose"1 :ai.onnx.contribgraphZ!
|
||||
|
||||
auth_token
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿZ
|
||||
X
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿZ
|
||||
Y
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿb
|
||||
Z
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿB
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
:Î
|
||||
Ó
|
||||
|
||||
auth_token
|
||||
X
|
||||
YZtriton_invoker"AzureTritonInvoker*
|
||||
|
||||
model_name"addi4 *G
|
||||
model_uri"7https://endpoint-9231153.westus2.inference.ml.azure.com *
|
||||
model_version"1 *
|
||||
verbose"1 :ai.onnx.contribgraphZ!
|
||||
|
||||
auth_token
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿZ
|
||||
X
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿZ
|
||||
Y
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿb
|
||||
Z
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿB
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
:Ì
|
||||
Ñ
|
||||
|
||||
auth_token
|
||||
X
|
||||
YZtriton_invoker"AzureTritonInvoker*
|
||||
|
||||
model_name"and *G
|
||||
model_uri"7https://endpoint-5911162.westus2.inference.ml.azure.com *
|
||||
model_version"1 *
|
||||
verbose"1 :ai.onnx.contribgraphZ!
|
||||
|
||||
auth_token
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿZ
|
||||
X
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿZ
|
||||
Y
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿb
|
||||
Z
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿB
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
:ò
|
||||
ä
|
||||
|
||||
auth_token
|
||||
str_instr_out1str_out2triton_invoker"AzureTritonInvoker*
|
||||
|
||||
model_name"str *G
|
||||
model_uri"7https://endpoint-7270363.westus2.inference.ml.azure.com *
|
||||
model_version"1 *
|
||||
verbose"1 :ai.onnx.contribgraphZ!
|
||||
|
||||
auth_token
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿZ
|
||||
str_in
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿb
|
||||
str_out1
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿb
|
||||
str_out2
|
||||
|
||||
ÿÿÿÿÿÿÿÿÿB
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
import os
|
||||
import unittest
|
||||
import numpy as np
|
||||
|
||||
from onnx import checker, helper, onnx_pb as onnx_proto
|
||||
from onnxruntime_extensions import PyOrtFunction, util, get_library_path
|
||||
from onnxruntime import *
|
||||
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
ort_ext_root = os.path.abspath(os.path.join(script_dir, ".."))
|
||||
test_data_dir = os.path.join(ort_ext_root, "test", "data", "azure")
|
||||
|
||||
class TestAzureOps(unittest.TestCase):
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
self.__enabled = os.getenv('OCOS_ENABLE_AZURE','') == '1'
|
||||
if self.__enabled:
|
||||
self.__opt = SessionOptions()
|
||||
self.__opt.register_custom_ops_library(get_library_path())
|
||||
|
||||
def test_addf(self):
|
||||
if self.__enabled:
|
||||
sess = InferenceSession(os.path.join(test_data_dir, "triton_addf.onnx"),
|
||||
self.__opt, providers=["CPUExecutionProvider"])
|
||||
auth_token = np.array([os.getenv('ADDF', '')])
|
||||
x = np.array([1,2,3,4]).astype(np.float32)
|
||||
y = np.array([4,3,2,1]).astype(np.float32)
|
||||
ort_inputs = {
|
||||
"auth_token": auth_token,
|
||||
"X": x,
|
||||
"Y": y
|
||||
}
|
||||
out = sess.run(None, ort_inputs)[0]
|
||||
self.assertTrue(np.allclose(out, [5,5,5,5]))
|
||||
|
||||
def testAddf8(self):
|
||||
if self.__enabled:
|
||||
opt = SessionOptions()
|
||||
opt.register_custom_ops_library(get_library_path())
|
||||
sess = InferenceSession(os.path.join(test_data_dir, "triton_addf8.onnx"),
|
||||
self.__opt, providers=["CPUExecutionProvider"])
|
||||
auth_token = np.array([os.getenv('ADDF8', '')])
|
||||
x = np.array([1,2,3,4]).astype(np.double)
|
||||
y = np.array([4,3,2,1]).astype(np.double)
|
||||
ort_inputs = {
|
||||
"auth_token": auth_token,
|
||||
"X": x,
|
||||
"Y": y
|
||||
}
|
||||
out = sess.run(None, ort_inputs)[0]
|
||||
self.assertTrue(np.allclose(out, [5,5,5,5]))
|
||||
|
||||
def testAddi4(self):
|
||||
if self.__enabled:
|
||||
sess = InferenceSession(os.path.join(test_data_dir, "triton_addi4.onnx"),
|
||||
self.__opt, providers=["CPUExecutionProvider"])
|
||||
auth_token = np.array([os.getenv('ADDI4', '')])
|
||||
x = np.array([1,2,3,4]).astype(np.int32)
|
||||
y = np.array([4,3,2,1]).astype(np.int32)
|
||||
ort_inputs = {
|
||||
"auth_token": auth_token,
|
||||
"X": x,
|
||||
"Y": y
|
||||
}
|
||||
out = sess.run(None, ort_inputs)[0]
|
||||
self.assertTrue(np.allclose(out, [5,5,5,5]))
|
||||
|
||||
def testAnd(self):
|
||||
if self.__enabled:
|
||||
sess = InferenceSession(os.path.join(test_data_dir, "triton_and.onnx"),
|
||||
self.__opt, providers=["CPUExecutionProvider"])
|
||||
auth_token = np.array([os.getenv('AND', '')])
|
||||
x = np.array([True, True])
|
||||
y = np.array([True, False])
|
||||
ort_inputs = {
|
||||
"auth_token": auth_token,
|
||||
"X": x,
|
||||
"Y": y
|
||||
}
|
||||
out = sess.run(None, ort_inputs)[0]
|
||||
self.assertTrue(np.allclose(out, [True, False]))
|
||||
|
||||
def testStr(self):
|
||||
if self.__enabled:
|
||||
sess = InferenceSession(os.path.join(test_data_dir, "triton_str.onnx"),
|
||||
self.__opt, providers=["CPUExecutionProvider"])
|
||||
auth_token = np.array([os.getenv('STR', '')])
|
||||
str_in = np.array(['this is the input'])
|
||||
ort_inputs = {
|
||||
"auth_token": auth_token,
|
||||
"str_in": str_in
|
||||
}
|
||||
outs = sess.run(None, ort_inputs)
|
||||
self.assertEqual(len(outs), 2)
|
||||
self.assertEqual(outs[0], ['this is the input'])
|
||||
self.assertEqual(outs[1], ['this is the input'])
|
||||
|
||||
def testOpenAiAudio(self):
|
||||
if self.__enabled:
|
||||
sess = InferenceSession(os.path.join(test_data_dir, "openai_audio.onnx"),
|
||||
self.__opt, providers=["CPUExecutionProvider"])
|
||||
auth_token = np.array([os.getenv('AUDIO', '')])
|
||||
model = np.array(['whisper-1'])
|
||||
response_format = np.array(['text'])
|
||||
|
||||
with open(os.path.join(test_data_dir, "test16.wav"), "rb") as _f:
|
||||
audio_blob = np.asarray(list(_f.read()), dtype=np.uint8)
|
||||
ort_inputs = {
|
||||
"auth_token": auth_token,
|
||||
"model": model,
|
||||
"response_format": response_format,
|
||||
"file": audio_blob
|
||||
}
|
||||
out = sess.run(None, ort_inputs)[0]
|
||||
self.assertEqual(out, ['This is a test recording to test the Whisper model.\n'])
|
||||
|
||||
def testOpenAiChat(self):
|
||||
if self.__enabled:
|
||||
sess = InferenceSession(os.path.join(test_data_dir, "openai_chat.onnx"),
|
||||
self.__opt, providers=["CPUExecutionProvider"])
|
||||
auth_token = np.array([os.getenv('CHAT', '')])
|
||||
chat = np.array(['{\"model\": \"gpt-3.5-turbo\",\"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}, {\"role\": \"user\", \"content\": \"Hello!\"}]}'])
|
||||
ort_inputs = {
|
||||
"auth_token": auth_token,
|
||||
"chat": chat,
|
||||
}
|
||||
out = sess.run(None, ort_inputs)[0]
|
||||
self.assertTrue('assist' in out[0])
|
||||
|
||||
def testOpenAiEmb(self):
|
||||
if self.__enabled:
|
||||
opt = SessionOptions()
|
||||
opt.register_custom_ops_library(get_library_path())
|
||||
sess = InferenceSession(os.path.join(test_data_dir, "openai_embedding.onnx"),
|
||||
opt, providers=["CPUExecutionProvider"])
|
||||
auth_token = np.array([os.getenv('EMB', '')])
|
||||
text = np.array(['{\"input\": \"The food was delicious and the waiter...\", \"model\": \"text-embedding-ada-002\"}'])
|
||||
|
||||
ort_inputs = {
|
||||
"auth_token": auth_token,
|
||||
"text": text,
|
||||
}
|
||||
|
||||
out = sess.run(None, ort_inputs)[0]
|
||||
self.assertTrue('text-embedding-ada' in out[0])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
@echo off
|
||||
if "%OCOS_ENABLE_AZURE%"=="1" (
|
||||
pushd %1\test
|
||||
python -m pip install onnxruntime
|
||||
python test_azure_ops.py
|
||||
popd
|
||||
)
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [[ "$OCOS_ENABLE_AZURE" == "1" ]]
|
||||
then
|
||||
pushd $1/test
|
||||
python -m pip install onnxruntime
|
||||
python ./test_azure_ops.py
|
||||
popd
|
||||
fi
|
||||
Загрузка…
Ссылка в новой задаче