From ee14fbe48ed0a15bb9fb6c7cacb4cd5158c48438 Mon Sep 17 00:00:00 2001 From: Wenbing Li <10278425+wenbingl@users.noreply.github.com> Date: Wed, 16 Aug 2023 12:51:17 -0700 Subject: [PATCH] correct CLIP tokenizer name (#526) --- nuget/NativeNuget.nuspec | 2 +- onnxruntime_extensions/_hf_cvt.py | 2 +- tools/android/package_ops.config | 2 +- tools/ios/package_ops.config | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nuget/NativeNuget.nuspec b/nuget/NativeNuget.nuspec index e79c95b5..8529ef8e 100644 --- a/nuget/NativeNuget.nuspec +++ b/nuget/NativeNuget.nuspec @@ -11,7 +11,7 @@ 2. Support for pre-processing and post-processing of the Whisper model, inclusive of Audio and Tokenizer decoding operators. 3. Extends support for pre-processing and post-processing of object-detection models, including a new DrawBoundingBoxes operator. Pre/post processing tools can add non-max-suppression to the model to select the best bounding boxes, and scale those to the original image. See the end-to-end example in yolo_e2e.py. 4. Introduces the Audio Domain, complemented with AudioCodec and optimized STFT Operators, enhancing audio processing capabilities. - 5. Enabled optional input/output support for some operators such as GPT2Tokenizer, ClipTokenizer, and RobertaTokenizer. + 5. Enabled optional input/output support for some operators such as GPT2Tokenizer, CLIPTokenizer, and RobertaTokenizer. 6. Refined the implementation of offset mapping for BBPE-style tokenizers for more operators and efficiency improvement. 7. Other bug and security fixes. diff --git a/onnxruntime_extensions/_hf_cvt.py b/onnxruntime_extensions/_hf_cvt.py index bbc090c3..581b88aa 100644 --- a/onnxruntime_extensions/_hf_cvt.py +++ b/onnxruntime_extensions/_hf_cvt.py @@ -135,7 +135,7 @@ _PROCESSOR_DICT = { 'BertDecoder', HFTokenizerConverter.bpe_decoder, None), "GPT2Tokenizer": TokenOpParam('GPT2Tokenizer', HFTokenizerConverter.bpe_tokenizer, 'BpeDecoder', HFTokenizerConverter.bpe_decoder, None), - "ClipTokenizer": TokenOpParam('ClipTokenizer', HFTokenizerConverter.clip_tokenizer, + "CLIPTokenizer": TokenOpParam('CLIPTokenizer', HFTokenizerConverter.clip_tokenizer, 'BpeDecoder', HFTokenizerConverter.bpe_decoder, None), "RobertaTokenizer": TokenOpParam("RobertaTokenizer", HFTokenizerConverter.roberta_tokenizer, None, None, None), diff --git a/tools/android/package_ops.config b/tools/android/package_ops.config index e6de589a..4d297dbf 100644 --- a/tools/android/package_ops.config +++ b/tools/android/package_ops.config @@ -1 +1 @@ -com.microsoft.extensions;1;DecodeImage,EncodeImage,BertTokenizer,ClipTokenizer,BpeDecoder,DrawBoundingBoxes,OpenAIAudioToText +com.microsoft.extensions;1;DecodeImage,EncodeImage,BertTokenizer,CLIPTokenizer,BpeDecoder,DrawBoundingBoxes,OpenAIAudioToText diff --git a/tools/ios/package_ops.config b/tools/ios/package_ops.config index 4f74dc5a..6e853011 100644 --- a/tools/ios/package_ops.config +++ b/tools/ios/package_ops.config @@ -1 +1 @@ -com.microsoft.extensions;1;DecodeImage,EncodeImage,BertTokenizer,ClipTokenizer,BpeDecoder,DrawBoundingBoxes +com.microsoft.extensions;1;DecodeImage,EncodeImage,BertTokenizer,CLIPTokenizer,BpeDecoder,DrawBoundingBoxes