From fdcc9bd849acf2a5faa296f7ee42e2d09b92c69a Mon Sep 17 00:00:00 2001 From: AmitMY Date: Mon, 27 Jul 2026 13:11:38 +0200 Subject: [PATCH] refactor(tokenizer): drop inert __slots__, inline single-use num_embeddings __slots__ listed three attributes but did nothing: no ancestor of PreTrainedTokenizer defines __slots__, so instances carry a __dict__ and the declaration saves no memory. Verified by walking the MRO for a __slots__ definition. It read as an optimization that was not one. CharacterEmbedding.num_embeddings was set and then read once, two lines later, to build the embedding; inlined the 256. Dropped the _encode / _encode_and_truncate merge that was in the first version of this branch: the split, and hoisting the attribute lookups out of the comprehensions, is deliberate for the hot tokenization path. Left a comment saying so, since nothing in the code said it and the two methods look like accidental duplication. Co-Authored-By: Claude Opus 5 (1M context) --- utf8_tokenizer/char_embeddings.py | 3 +-- utf8_tokenizer/tokenizer.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/utf8_tokenizer/char_embeddings.py b/utf8_tokenizer/char_embeddings.py index 7e7b5f9..0201cd5 100644 --- a/utf8_tokenizer/char_embeddings.py +++ b/utf8_tokenizer/char_embeddings.py @@ -43,9 +43,8 @@ def __init__(self, embedding_size: int, num_bytes: int): self.embedding_size = embedding_size self.num_bytes = num_bytes self.byte_dim = embedding_size // num_bytes - self.num_embeddings = 256 - self.embedding = nn.Embedding(self.num_embeddings, self.byte_dim) + self.embedding = nn.Embedding(256, self.byte_dim) with torch.no_grad(): # Initialize embedding weights with normalized rows for roundtrip diff --git a/utf8_tokenizer/tokenizer.py b/utf8_tokenizer/tokenizer.py index fac21e1..ac13487 100644 --- a/utf8_tokenizer/tokenizer.py +++ b/utf8_tokenizer/tokenizer.py @@ -53,8 +53,6 @@ class UTFTokenizer(PreTrainedTokenizer): Exposes a `.torch` method for optimized tensor creation. """ - __slots__ = ('_struct_format', '_bytes_per_unit', '_eos_encoded') - encoding: str = "utf-8" dtype: torch.dtype = torch.uint8 @@ -146,6 +144,8 @@ def _encode(self, texts: list[TextInput], add_special_tokens: bool) -> list[byte def _encode_and_truncate( self, texts: list[TextInput], max_length: int, add_special_tokens: bool ) -> list[bytes]: + # Kept separate from _encode, and hoisting these lookups out of the comprehensions, + # because this is the hot tokenization path encoding, bytes_per_unit, eos_encoded = self.encoding, self._bytes_per_unit, self._eos_encoded # Convert from token count to byte count for slicing max_length *= bytes_per_unit