add CLIPTokenizer test for AutoTokenizer (#537)

Co-authored-by: Sayan Shaw <sayanshaw@microsoft.com>

test/test_autotokenizer.py

@@ -29,7 +29,6 @@ class TestAutoTokenizer(unittest.TestCase):
         np.testing.assert_array_equal(ids[0], actual_ids)
 
     def test_falcon_tokenizer(self):
-        # replace the official model name after the model is not gated anymore
         tokenizer = AutoTokenizer.from_pretrained("Rocketknight1/falcon-rw-1b", use_fast=False)
         text = "why don't you teach me some German?"
         ids = tokenizer.encode(text, return_tensors="np")
@@ -59,6 +58,17 @@ class TestAutoTokenizer(unittest.TestCase):
 
         self.assertEqual(OrtPyFunction(m_detok)(ids)[0], tokenizer.decode(ids[0]))
 
+    def test_clip_tokenizer(self):
+        tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32", use_fast=False)
+        text = "Wow, these models are getting popular."
+        ids = tokenizer.encode(text, return_tensors="np")
+
+        ort_tok = OrtPyFunction.from_model(gen_processing_models(
+            tokenizer,
+            pre_kwargs={"WITH_DEFAULT_INPUTS": True})[0])
+        actual_ids = ort_tok([text])[0]
+        np.testing.assert_array_equal(ids, actual_ids)
+
 
 if __name__ == '__main__':
     unittest.main()