_encode_plus → get_input_ids:
    → tokenize(text):
        → tokens_trie.split(text)                 ← 76% of overhead: O(n*k) trie scan
        → _tokenize(segment):
            → self.encode(segment) [tiktoken/Rust → list[int]]
            → [self.decoder[t] for t ...]          ← 3%: int→str
    → convert_tokens_to_ids(strings)               ← 21%: str→int
→ prepare_for_model(ids, truncation, special tokens)

---

vllm/renderers/base.py:399         await tokenizer.encode(prompt, **kwargs)
vllm/utils/async_utils.py:63       return (await self(prompt, **kwargs)).input_ids
  → async_utils.py:118               self.tokenizer(text, **kwargs)       # __call__
  → tokenization_utils_base.py:2996  __call__ → encode_plus
  → tokenization_utils.py:743        _encode_plus():
      → line 767                        tokens = self.tokenize(text)       # slow chain starts
      → tokenization_utils.py:661         tokens_trie.split(text)          # 76% overhead
      → tokenization_utils.py:697         self._tokenize(segment)
      → tokenization_kimi.py:283            [self.decoder[t] for t in self.encode(text)]
                                            # encode(text) → fast tiktoken → list[int]
                                            # then int→str for each token (3%)
      → line 768                        self.convert_tokens_to_ids(tokens)  # str→int (21%)
      → line 803                        self.prepare_for_model(ids, ...)    # truncation etc.

---

def _encode_plus(self, text, text_pair=None, add_special_tokens=True,
                 padding_strategy=PaddingStrategy.DO_NOT_PAD,
                 truncation_strategy=TruncationStrategy.DO_NOT_TRUNCATE,
                 max_length=None, stride=0, is_split_into_words=False,
                 pad_to_multiple_of=None, padding_side=None,
                 return_tensors=None, return_token_type_ids=None,
                 return_attention_mask=None, return_overflowing_tokens=False,
                 return_special_tokens_mask=False, return_offsets_mapping=False,
                 return_length=False, verbose=True, **kwargs):
    # Fast path: call self.encode() directly → tiktoken (Rust).
    # Bypasses tokenize() → _tokenize() → convert_tokens_to_ids() overhead.
    if isinstance(text, str) and text_pair is None and not is_split_into_words:
        first_ids = self.encode(text)
    else:
        return super()._encode_plus(
            text, text_pair=text_pair, add_special_tokens=add_special_tokens,
            padding_strategy=padding_strategy, truncation_strategy=truncation_strategy,
            max_length=max_length, stride=stride, is_split_into_words=is_split_into_words,
            pad_to_multiple_of=pad_to_multiple_of, padding_side=padding_side,
            return_tensors=return_tensors, return_token_type_ids=return_token_type_ids,
            return_attention_mask=return_attention_mask,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_offsets_mapping=return_offsets_mapping,
            return_length=return_length, verbose=verbose, **kwargs)

    return self.prepare_for_model(
        first_ids, add_special_tokens=add_special_tokens,
        padding=padding_strategy.value, truncation=truncation_strategy.value,
        max_length=max_length, stride=stride,
        pad_to_multiple_of=pad_to_multiple_of, padding_side=padding_side,
        return_tensors=return_tensors, prepend_batch_axis=True,
        return_attention_mask=return_attention_mask,
        return_token_type_ids=return_token_type_ids,
        return_overflowing_tokens=return_overflowing_tokens,
        return_special_tokens_mask=return_special_tokens_mask,
        return_length=return_length, verbose=verbose)

---

import time
from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("nvidia/Kimi-K2.5-NVFP4", trust_remote_code=True)
prompt = "The quick brown fox jumps over the lazy dog. " * 1300

for _ in range(3):  # warmup
    tokenizer.encode(prompt)
    tokenizer(prompt, add_special_tokens=True, truncation=True, max_length=131073)

t0 = time.perf_counter()
fast = tokenizer.encode(prompt)
t_fast = time.perf_counter() - t0

t0 = time.perf_counter()
slow = tokenizer(prompt, add_special_tokens=True, truncation=True, max_length=131073).input_ids
t_slow = time.perf_counter() - t0

print(f"encode():  {t_fast*1000:.1f} ms | __call__(): {t_slow*1000:.1f} ms | {t_slow/t_fast:.1f}x | match={fast==slow}")