Tokenizer

bos_token: str | None

String representation of the BOS token.

>>> tokenizer.bos_token  # Llama 3.2
'<|begin_of_text|>'

bos_token_id: int | None

Beginning-of-sequence token ID.

Returns None if the model doesn't use a BOS token (e.g., Qwen3).

>>> tokenizer.bos_token_id  # Llama 3.2
128000
>>> tokenizer.bos_token_id  # Qwen3
None

eos_token_ids: tuple[int, ...]

Token IDs that signal end of generation.

Returns an immutable, ordered tuple of deduplicated EOS token IDs. Many models have multiple EOS tokens (e.g., Qwen, Llama 3, Gemma 3).

>>> tokenizer.eos_token_ids
(151643, 151644, 151645)

eos_tokens: list[str]

String representations of all EOS tokens.

>>> tokenizer.eos_tokens
['<|endoftext|>', '<|im_end|>', '<|end|>']

model_max_length: int

Maximum sequence length the model supports.

This value is read from tokenizer_config.json (model_max_length field). It represents the maximum context length the model was trained with.

When truncation=True is used without an explicit max_length, this value is used as the default truncation limit.

Returns 0 if the model does not specify a maximum length.

>>> tokenizer.model_max_length
32768

model_path: str

The resolved path to the model directory.

pad_token: str | None

String representation of the padding token.

pad_token_id: int | None

Padding token ID.

padding_side: str

Side for padding: "left" (default, for generation) or "right".

special_ids: frozenset[int]

Immutable set of all special token IDs.

Includes EOS, BOS, UNK, and PAD tokens. Use for fast O(1) membership testing.

>>> 128001 in tokenizer.special_ids
True

truncation_side: str

Side for truncation: "right" (default, keeps beginning) or "left" (keeps end).

unk_token: str | None

String representation of the unknown token.

unk_token_id: int | None

Unknown token ID.

vocab_size: int

Number of tokens in the vocabulary.

def __call__(
    self,
    text: str | list[str],
    text_pair: str | list[str] | None = None,
    special_tokens: bool | set[str] = True,
    truncation: bool | str = False,
    max_length: int | None = None,
    return_tensors: str | None = None,
    kwargs: Any
) → BatchEncoding

Callable interface for tokenization with zero-copy tensor access.

Returns a BatchEncoding object that provides dict-like access to input_ids and attention_mask via the DLPack protocol. This enables zero-copy transfer to PyTorch, JAX, or NumPy.

Padding is applied automatically when exporting to tensors. Control the padding side via tokenizer.padding_side or batch.padding_side.

>>> batch = tokenizer(["Hello", "World!"])
>>> input_ids = torch.from_dlpack(batch["input_ids"])
>>> attention_mask = torch.from_dlpack(batch["attention_mask"])
>>> model(input_ids, attention_mask=attention_mask)

def apply_chat_template(
    self,
    messages: list,
    add_generation_prompt: bool = True,
    tokenize: bool = False
) → str | TokenArray | BatchEncoding

Format a conversation using the model's chat template.

def close(self) → None

Release native tokenizer resources.

After calling close(), the tokenizer cannot be used. Safe to call multiple times (idempotent).

def convert_ids_to_tokens(self, ids: list[int]) → list[str | None]

Convert a list of token IDs to their string representations.

def convert_tokens_to_ids(self, tokens: list[str]) → list[int | None]

Convert a list of token strings to their IDs.

def count_tokens(
    self,
    text: str,
    special_tokens: bool | set[str] = True
) → int

Count the number of tokens in text.

This returns the exact token count that would be used in generation, including BOS/EOS tokens by default. Use this to check if prompts fit within context windows.

>>> # Check if prompt fits in context window
>>> tokens = tokenizer.count_tokens(prompt)
>>> if tokens > 4096:
...     print("Prompt too long!")

>>> # Count without special tokens
>>> content_tokens = tokenizer.count_tokens(text, special_tokens=False)

def decode(
    self,
    tokens: TokenArray | list[int],
    num_tokens: int | None = None,
    skip_special_tokens: bool = True
) → str

Convert token IDs back to text.

def encode(
    self,
    text: str | list[str],
    special_tokens: bool | set[str] = True,
    truncation: bool = False,
    max_length: int | None = None,
    truncation_side: str | None = None
) → TokenArray | BatchEncoding

Convert text to token IDs.

batch = tokenizer.encode(["Hello", "World"])
batch.padding_side = "right"  # Override before converting
tensor = torch.from_dlpack(batch)

>>> # Default: add all special tokens
>>> tokens = tokenizer.encode("Hello world")

>>> # No special tokens
>>> tokens = tokenizer.encode("Hello world", special_tokens=False)

>>> # BOS only (useful for document snippets)
>>> tokens = tokenizer.encode("Document...", special_tokens={"bos"})

>>> # With truncation (uses model_max_length if no explicit max_length)
>>> tokens = tokenizer.encode(long_text, truncation=True)

>>> # With explicit max_length
>>> tokens = tokenizer.encode(text, truncation=True, max_length=512)

>>> # Left truncation for RAG (keep recent context)
>>> tokens = tokenizer.encode(text, truncation=True, max_length=512,
...                           truncation_side="left")

>>> # Check if model has BOS token
>>> if tokenizer.bos_token_id is not None:
...     print(f"BOS token: {tokenizer.bos_token}")

def from_json(
    cls,
    json_content: str | bytes,
    chat_template: str | None = None,
    bos_token: str = '',
    eos_token: str = '',
    padding_side: str = 'left',
    truncation_side: str = 'right'
) → Tokenizer

Create a tokenizer directly from JSON content.

Creates a standalone tokenizer without needing a model directory. Useful for custom tokenizers, serverless deployments, or testing.

>>> json = '{"version": "1.0", "model": {"type": "BPE", ...}}'
>>> template = "{% for m in messages %}{{ m.content }}{% endfor %}"
>>> tok = Tokenizer.from_json(json, chat_template=template)
>>> prompt = tok.apply_chat_template([{"role": "user", "content": "Hi"}])

def get_vocab(self) → dict[str, int]

Get the complete vocabulary as a dictionary.

def id_to_token(self, token_id: int) → str | None

Get the string representation of a token ID.

>>> tokenizer.id_to_token(9707)
'Hello'

def is_special_id(self, token_id: int) → bool

Check if a token ID is a special token.

>>> tokenizer.is_special_id(128001)  # EOS token
True

def primary_eos_token_id(self) → int | None

Get the primary EOS token ID for insertion.

Use this when inserting an EOS token. For detection/stopping, use token_id in eos_token_ids instead.

>>> eos_id = tokenizer.primary_eos_token_id()
>>> tokens.append(eos_id)

def token_to_id(self, token: str) → int | None

Get the ID of a token string.

>>> tokenizer.token_to_id('Hello')
9707

def tokenize(
    self,
    text: str,
    return_bytes: bool = False
) → list[str] | list[bytes]

Split text into token strings.

This is useful for debugging tokenization - seeing exactly how text is segmented before being converted to token IDs.

>>> tokenizer.tokenize("Hello world")
['Hello', ' world']

>>> # Debug mode - see raw bytes
>>> tokenizer.tokenize("Hello", return_bytes=True)
[b'Hello']

>>> # Useful for debugging unicode edge cases
>>> tokenizer.tokenize("café", return_bytes=True)
[b'caf', b'\\xc3\\xa9']  # Shows UTF-8 encoding

offsets: list[TokenOffset]

Lazy byte offsets mapping tokens back to source text.

Each offset is a (start, end) pair of UTF-8 byte indices into the original text. The first access triggers computation via the Zig runtime; subsequent accesses return the cached result.

Special tokens (BOS, EOS, PAD) that don't correspond to source text are assigned (0, 0).

>>> text = "Hello 🎉 world"
>>> tokens = tokenizer.encode(text)
>>> tokens.offsets
[(0, 5), (5, 10), ...]

>>> tokens.offsets[0].slice(text)
'Hello'
>>> tokens.offsets[-1].slice(text)
' world'

def close(self) → None

Release the underlying native buffer.

After calling close(), the TokenArray cannot be used for data access, DLPack export, or NumPy conversion. Safe to call multiple times (idempotent).

def count(self, value: int) → int

Return number of occurrences of value.

>>> tokens = tokenizer.encode("hello hello hello")
>>> tokens.count(9707)  # Count occurrences
3

def index(
    self,
    value: int,
    start: int = 0,
    stop: int | None = None
) → int

Return index of first occurrence of value.

>>> tokens = tokenizer.encode("Hello world")
>>> tokens.index(9707)  # Find first occurrence
0

def tolist(self) → list[int]

Convert to a Python list.

This copies the data into a new Python list. Use this when you need a regular list, or when you need to keep the data after the TokenArray is deleted.

>>> tokens = tokenizer.encode("Hello world")
>>> token_list = tokens.tolist()
>>> print(token_list)
[9707, 1879]
>>> type(token_list)
<class 'list'>

def count(self, value: int) → int

Return number of occurrences of value.

def index(
    self,
    value: int,
    start: int = 0,
    stop: int | None = None
) → int

Return index of first occurrence of value.

def tolist(self) → list[int]

Convert to a Python list (copies data).

def slice(
    self,
    text: str | bytes,
    errors: str = 'strict'
) → str

Extract the text span corresponding to this offset.

This is the recommended way to use offsets. It handles the conversion between byte offsets and Python string indexing automatically.

>>> tokens = tokenizer.encode("Hello 🎉 world")
>>> tokens.offsets[-1].slice("Hello 🎉 world")
' world'

>>> tokens.offsets[1].slice(text, errors="replace")
' \ufffd\ufffd'  # Replacement characters for partial bytes

attention_mask: _DLPackAccessor

DLPack interface for the attention mask tensor.

• Returns an accessor object that implements the DLPack protocol. Use with torch.from_dlpack() or np.from_dlpack() to get a 2D tensor of shape (num_sequences, padded_length) with values:
• 1 for real tokens
• 0 for padding tokens

NOTE: Each export allocates a new mask buffer computed from sequence lengths and padding configuration.

>>> batch = tokenizer.encode(["Hello", "World!"])
>>> input_ids = torch.from_dlpack(batch.input_ids)
>>> attention_mask = torch.from_dlpack(batch.attention_mask)
>>> model(input_ids, attention_mask=attention_mask)

input_ids: _DLPackAccessor

DLPack interface for the input_ids tensor.

Returns an accessor object that implements the DLPack protocol. Use with torch.from_dlpack() or np.from_dlpack() to get a 2D tensor of shape (num_sequences, padded_length).

NOTE: Each export allocates a new padded buffer. The internal CSR storage is materialized to dense format for ML framework consumption.

>>> batch = tokenizer.encode(["Hello", "World!"])
>>> input_ids = torch.from_dlpack(batch.input_ids)
>>> attention_mask = torch.from_dlpack(batch.attention_mask)
>>> model(input_ids, attention_mask=attention_mask)

pad_token_id: int | None

Padding token ID used for this batch.

padding_side: str

Side for padding: "left" (default for generation) or "right".

This property is inherited from the tokenizer when the batch is created. Set this property to override padding behavior before calling to_list() or using torch.from_dlpack().

>>> batch = tokenizer.encode(["Hello", "World"])
>>> batch.padding_side
'left'
>>> batch.padding_side = "right"  # Override for this batch
>>> tensor = torch.from_dlpack(batch)  # Uses right padding

total_tokens: int

Total number of tokens across all sequences.

def close(self) → None

Release the native batch encoding memory.

After calling close(), the BatchEncoding cannot be used for data access, DLPack export, or iteration. Safe to call multiple times (idempotent).

def keys(self) → list[str]

Return available tensor keys (dict-like interface).

def lengths(self) → list[int]

Get the length of each sequence in the batch.

def max_length(self) → int

Get the maximum sequence length in the batch.

def to_list(
    self,
    padding: bool = True,
    pad_id: int | None = None,
    padding_side: str | None = None,
    max_length: int | None = None,
    truncation: bool = False,
    return_attention_mask: bool = True
) → dict[str, list[list[int]]]

Convert batch to padded Python lists.

This is primarily useful for debugging or when you need plain Python data structures. For ML workloads, use torch.from_dlpack(batch.input_ids) and torch.from_dlpack(batch.attention_mask) directly.