Add TiktokenTokenizer (cl100k_base wrapper)

tokenization_parchment.py

@@ -22,6 +22,14 @@ class TiktokenTokenizer(PreTrainedTokenizer):
         self._build_enc()
 
         eot_str = self._id_to_tok[self._enc.eot_token]  # "<|endoftext|>"
+
+        # When loading from saved config, special tokens are already in kwargs —
+        # pop them so we don't pass duplicates to super().__init__().
+        kwargs.pop("bos_token", None)
+        kwargs.pop("eos_token", None)
+        kwargs.pop("pad_token", None)
+        kwargs.pop("unk_token", None)
+
         super().__init__(
             encoding_name=encoding_name,
             bos_token=eot_str,