refactor(tokenizer): drop inert __slots__, inline single-use num_embeddings

utf8_tokenizer/char_embeddings.py

@@ -43,9 +43,8 @@ def __init__(self, embedding_size: int, num_bytes: int):
         self.embedding_size = embedding_size
         self.num_bytes = num_bytes
         self.byte_dim = embedding_size // num_bytes
-        self.num_embeddings = 256
 
-        self.embedding = nn.Embedding(self.num_embeddings, self.byte_dim)
+        self.embedding = nn.Embedding(256, self.byte_dim)
 
         with torch.no_grad():
             # Initialize embedding weights with normalized rows for roundtrip

utf8_tokenizer/tokenizer.py

@@ -53,8 +53,6 @@ class UTFTokenizer(PreTrainedTokenizer):
     Exposes a `.torch` method for optimized tensor creation.
     """
 
-    __slots__ = ('_struct_format', '_bytes_per_unit', '_eos_encoded')
-
     encoding: str = "utf-8"
     dtype: torch.dtype = torch.uint8
 
@@ -146,6 +144,8 @@ def _encode(self, texts: list[TextInput], add_special_tokens: bool) -> list[byte
     def _encode_and_truncate(
         self, texts: list[TextInput], max_length: int, add_special_tokens: bool
     ) -> list[bytes]:
+        # Kept separate from _encode, and hoisting these lookups out of the comprehensions,
+        # because this is the hot tokenization path
         encoding, bytes_per_unit, eos_encoded = self.encoding, self._bytes_per_unit, self._eos_encoded
         # Convert from token count to byte count for slicing
         max_length *= bytes_per_unit