Fix CLIPTokenizer inconsistency between Python and TF paths (#2732)

Author: buildwithsuhana

keras_hub/src/models/clip/clip_tokenizer.py

@@ -1,11 +1,8 @@
-import inspect
-
 import tokenizers
 from tokenizers import decoders
 from tokenizers import models
 from tokenizers import normalizers
 from tokenizers import pre_tokenizers
-from tokenizers import processors
 
 from keras_hub.src.api_export import keras_hub_export
 from keras_hub.src.models.clip.clip_backbone import CLIPBackbone
@@ -153,24 +150,6 @@ def set_vocabulary_and_merges(self, vocabulary, merges):
         super().set_vocabulary_and_merges(vocabulary, merges)
         if self.pad_with_end_token:
             self.pad_token_id = self.end_token_id
-        if getattr(self, "_tokenizer") is not None:
-            # tokenizers <=0.22 use `cls`, >= 0.23 use `cls_token` because `cls`
-            # collides with the first argument of the Python `__new__`.
-            cls_token_arg = (
-                "cls"
-                if "cls"
-                in inspect.signature(processors.RobertaProcessing).parameters
-                else "cls_token"
-            )
-            preprocessing_args = {
-                "sep": (str(self.end_token), self.end_token_id),
-                cls_token_arg: (str(self.start_token), self.start_token_id),
-                "add_prefix_space": False,
-                "trim_offsets": False,
-            }
-            self._tokenizer.post_processor = processors.RobertaProcessing(
-                **preprocessing_args
-            )
 
     def _bpe_merge_and_update_cache_tf(self, tokens):
         """Process unseen tokens and add to cache."""
@@ -254,28 +233,16 @@ def process_unseen_tokens():
         if self.sequence_length:
             output_shape = tokens.shape.as_list()
             output_shape[-1] = self.sequence_length
-            tokens = tokens.to_tensor(shape=output_shape)
+            tokens = tokens.to_tensor(
+                shape=output_shape, default_value=self.pad_token_id
+            )
 
         # Convert to a dense output if input in scalar
         if unbatched:
             tokens = tf.squeeze(tokens, 0)
             tf.ensure_shape(tokens, shape=[self.sequence_length])
         return tokens
 
-    def _tokenize_tokenizers(self, inputs):
-        outputs = super()._tokenize_tokenizers(inputs)
-        is_batched = True
-        if isinstance(outputs, str):
-            is_batched = False
-            outputs = [outputs]
-        elif isinstance(outputs, list) and isinstance(outputs[0], int):
-            is_batched = False
-            outputs = [outputs]
-        outputs = [output[1:-1] for output in outputs]
-        if not is_batched:
-            outputs = outputs[0]
-        return outputs
-
     @preprocessing_function
     def _detokenize_tf(self, inputs):
         self._check_vocabulary()

keras_hub/src/models/clip/clip_tokenizer_test.py

@@ -1,4 +1,5 @@
 import pytest
+import tensorflow as tf
 
 from keras_hub.src.models.clip.clip_tokenizer import CLIPTokenizer
 from keras_hub.src.tests.test_case import TestCase
@@ -35,6 +36,26 @@ def test_pad_with_end_token(self):
         tokenizer = CLIPTokenizer(**init_kwargs)
         self.assertEqual(tokenizer.pad_token_id, tokenizer.end_token_id)
 
+    def test_python_tf_consistency(self):
+        init_kwargs = self.init_kwargs.copy()
+        init_kwargs["sequence_length"] = 10
+        init_kwargs["pad_with_end_token"] = True
+        tokenizer = CLIPTokenizer(**init_kwargs)
+        input_data = ["airplane", "airplane airport"]
+
+        # Python workflow
+        python_output = tokenizer(input_data)
+
+        # TF workflow
+        ds = tf.data.Dataset.from_tensor_slices(input_data)
+        ds = ds.map(tokenizer)
+        tf_outputs = list(ds.as_numpy_iterator())
+
+        self.assertAllEqual(python_output, tf_outputs)
+        self.assertAllEqual(python_output[0][-1], tokenizer.pad_token_id)
+        self.assertAllEqual(tf_outputs[0][-1], tokenizer.pad_token_id)
+        self.assertAllEqual(tokenizer.pad_token_id, tokenizer.end_token_id)
+
     def test_errors_missing_special_tokens(self):
         with self.assertRaises(ValueError):
             CLIPTokenizer(vocabulary={"foo": 0, "bar": 1}, merges=["fo o"])