跳转至

llm.tokenization — Tokenizer Implementations

Support for HuggingFace tokenizers and a built-in character-level tokenizer for simple experiments.

Overview

Module Purpose
tokenizer Base tokenizer interface
simple_tokenizer Character-level tokenizer
bpe_tokenizer Byte-Pair Encoding tokenizer
train_bpe BPE training utilities

Base Tokenizer

tokenizer

BaseTokenizer

Bases: Protocol

Abstract base class for all tokenizers.

源代码位于： src/llm/tokenization/tokenizer.py 
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
class BaseTokenizer(Protocol):
    """
    Abstract base class for all tokenizers.
    """

    vocab_size: int
    pad_token_id: int | None
    bos_token_id: int | None
    eos_token_id: int | None

    def encode(self, text: str) -> list[int]:
        """Encodes a string into a list of token IDs."""
        ...

    def decode(self, tokens: list[int]) -> str:
        """Decodes a list of token IDs back into a string."""
        ...

encode 

encode(text)

Encodes a string into a list of token IDs.

源代码位于： src/llm/tokenization/tokenizer.py 
19
20
21
def encode(self, text: str) -> list[int]:
    """Encodes a string into a list of token IDs."""
    ...

decode 

decode(tokens)

Decodes a list of token IDs back into a string.

源代码位于： src/llm/tokenization/tokenizer.py 
23
24
25
def decode(self, tokens: list[int]) -> str:
    """Decodes a list of token IDs back into a string."""
    ...

HFTokenizer

Wrapper for HuggingFace Transformers Tokenizers.

源代码位于： src/llm/tokenization/tokenizer.py 
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
class HFTokenizer:
    """
    Wrapper for HuggingFace Transformers Tokenizers.
    """

    def __init__(self, model_name_or_path: str):
        from transformers import AutoTokenizer

        self._tokenizer: PreTrainedTokenizerBase = cast(
            "PreTrainedTokenizerBase",
            AutoTokenizer.from_pretrained(model_name_or_path),
        )

        # Ensure pad token exists
        if self._tokenizer.pad_token is None and isinstance(self._tokenizer.eos_token, str):
            self._tokenizer.pad_token = self._tokenizer.eos_token

    @property
    def vocab_size(self) -> int:
        return self._tokenizer.vocab_size

    @property
    def pad_token_id(self) -> int | None:
        return self._tokenizer.pad_token_id

    @property
    def bos_token_id(self) -> int | None:
        return self._tokenizer.bos_token_id

    @property
    def eos_token_id(self) -> int | None:
        return self._tokenizer.eos_token_id

    def encode(self, text: str) -> list[int]:
        # Return simple list of ints
        return self._tokenizer.encode(text, add_special_tokens=False)

    def decode(self, tokens: list[int]) -> str:
        decoded = self._tokenizer.decode(tokens)
        assert isinstance(decoded, str)  # noqa: S101
        return decoded

    def save_pretrained(self, save_directory: str):
        self._tokenizer.save_pretrained(save_directory)

    @classmethod
    def from_pretrained(cls, path: str) -> HFTokenizer:
        return cls(path)

Simple Character Tokenizer

simple_tokenizer

SimpleCharacterTokenizer

A simple character-level tokenizer.

This tokenizer builds a vocabulary from a given corpus and provides methods to encode text into a sequence of integer tokens and decode a sequence of tokens back into text.

源代码位于： src/llm/tokenization/simple_tokenizer.py 
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
class SimpleCharacterTokenizer:
    """
    A simple character-level tokenizer.

    This tokenizer builds a vocabulary from a given corpus and provides methods
    to encode text into a sequence of integer tokens and decode a sequence of
    tokens back into text.
    """

    pad_char: str = "<PAD>"
    eos_char: str = "<EOS>"
    bos_char: str = "<BOS>"

    def __init__(self, corpus: list[str]):
        """
        Initializes the SimpleCharacterTokenizer.

        Args:
            corpus (list[str]): A list of strings from which to build the vocabulary.
                                The vocabulary will consist of all unique characters
                                present in the corpus.
        """
        if not isinstance(corpus, list):
            raise TypeError("Corpus must be a list of strings.")
        if not all(isinstance(s, str) for s in corpus):
            raise TypeError("All items in the corpus must be strings.")

        # Join all strings in the corpus, then find unique characters
        unique_chars: set[str] = set("".join(corpus))
        self.chars: list[str] = sorted(unique_chars)  # Sort for consistent mapping

        self.stoi: dict[str, int] = {char: i for i, char in enumerate(self.chars)}
        self.itos: dict[int, str] = dict(enumerate(self.chars))
        self.vocab_size: int = len(self.chars)

        # Add PAD token
        if self.pad_char not in self.stoi:
            pad_token_id = self.vocab_size
            self.stoi[self.pad_char] = pad_token_id
            self.itos[pad_token_id] = self.pad_char
            self.chars.append(self.pad_char)  # Add to the list of characters
            self.vocab_size += 1
        else:
            # If PAD char was part of the corpus, use its existing ID
            pad_token_id = self.stoi[self.pad_char]

        self.pad_token_id: int = pad_token_id

        # ``<EOS>`` / ``<BOS>`` markers, declared alongside ``<PAD>`` by the
        # shared default corpus (``DEFAULT_SIMPLE_CORPUS``) and by
        # ``TokenizerFactory.from_dataset_text``. They are registered as real
        # multi-char special tokens *only when the corpus lists them*: without
        # this, the markers were flattened into their constituent plain
        # characters and ``eos_token_id`` stayed ``None``, so eval generations
        # (LMTask / lm_eval generate_until) never stopped on the model's EOS
        # (RIL ISS-152). Corpora that don't declare them keep the exact same
        # vocab layout as before.
        self._bos_token_id: int | None = None
        self._eos_token_id: int | None = None
        for marker, attr in ((self.eos_char, "_eos_token_id"), (self.bos_char, "_bos_token_id")):
            if marker in corpus and marker not in self.stoi:
                special_id = self.vocab_size
                self.stoi[marker] = special_id
                self.itos[special_id] = marker
                self.chars.append(marker)
                self.vocab_size += 1
                setattr(self, attr, special_id)

    @property
    def bos_token_id(self) -> int | None:
        return self._bos_token_id

    @property
    def eos_token_id(self) -> int | None:
        return self._eos_token_id

    def encode(self, text: str) -> list[int]:
        """
        Encodes a string of text into a list of integer tokens.

        Args:
            text (str): The input string to encode.

        Returns:
            list[int]: A list of integer tokens representing the input text.

        Raises:
            KeyError: If the text contains characters not present in the
                      tokenizer's vocabulary (i.e., not found in the
                      initial corpus).
        """
        if not isinstance(text, str):
            raise TypeError("Input text must be a string.")

        # A special marker encoded verbatim maps to its single token id (the
        # pad precedent). Without this, ``encode("<EOS>")`` flattened to the
        # char ids of '<','E','O','S','>' even when the tokenizer had
        # registered the marker as a special (RIL ISS-152). The guard keeps
        # the fast path for declared markers only: a tokenizer built from a
        # corpus that never registers EOS/BOS (e.g. a printable-only one)
        # would otherwise raise a bare ``KeyError: '<EOS>'`` with no context
        # (RIL ISS-214) — it now falls through to the char-wise loop, which
        # returns the literal composition or the contextual vocab error.
        if text in (self.pad_char, self.eos_char, self.bos_char) and text in self.stoi:
            return [self.stoi[text]]

        tokens: list[int] = []
        for char in text:
            try:
                tokens.append(self.stoi[char])
            except KeyError:
                raise KeyError(
                    f"Character '{char}' not found in tokenizer vocabulary. "
                    "Only characters present in the initial corpus can be encoded."
                )
        return tokens

    def decode(self, tokens: list[int]) -> str:
        """
        Decodes a list of integer tokens back into a string of text.

        Args:
            tokens (list[int]): A list of integer tokens to decode.

        Returns:
            str: The decoded string.

        Raises:
            KeyError: If the list contains token IDs not present in the
                      tokenizer's vocabulary.
        """
        if not isinstance(tokens, list):
            raise TypeError("Input tokens must be a list of integers.")
        if not all(isinstance(token, int) for token in tokens):
            raise TypeError("All items in the tokens list must be integers.")

        text_chars: list[str] = []
        for token in tokens:
            try:
                text_chars.append(self.itos[token])
            except KeyError:
                raise KeyError(
                    f"Token ID '{token}' not found in tokenizer vocabulary. "
                    "Only token IDs derived from the initial corpus can be decoded."
                )
        return "".join(text_chars)

encode 

encode(text)

Encodes a string of text into a list of integer tokens.

参数：

名称 类型 描述 默认
text str

The input string to encode.

 必需

返回：

类型 描述
list[int]

list[int]: A list of integer tokens representing the input text.

引发：

类型 描述
KeyError

If the text contains characters not present in the tokenizer's vocabulary (i.e., not found in the initial corpus).
源代码位于： src/llm/tokenization/simple_tokenizer.py 
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
def encode(self, text: str) -> list[int]:
    """
    Encodes a string of text into a list of integer tokens.

    Args:
        text (str): The input string to encode.

    Returns:
        list[int]: A list of integer tokens representing the input text.

    Raises:
        KeyError: If the text contains characters not present in the
                  tokenizer's vocabulary (i.e., not found in the
                  initial corpus).
    """
    if not isinstance(text, str):
        raise TypeError("Input text must be a string.")

    # A special marker encoded verbatim maps to its single token id (the
    # pad precedent). Without this, ``encode("<EOS>")`` flattened to the
    # char ids of '<','E','O','S','>' even when the tokenizer had
    # registered the marker as a special (RIL ISS-152). The guard keeps
    # the fast path for declared markers only: a tokenizer built from a
    # corpus that never registers EOS/BOS (e.g. a printable-only one)
    # would otherwise raise a bare ``KeyError: '<EOS>'`` with no context
    # (RIL ISS-214) — it now falls through to the char-wise loop, which
    # returns the literal composition or the contextual vocab error.
    if text in (self.pad_char, self.eos_char, self.bos_char) and text in self.stoi:
        return [self.stoi[text]]

    tokens: list[int] = []
    for char in text:
        try:
            tokens.append(self.stoi[char])
        except KeyError:
            raise KeyError(
                f"Character '{char}' not found in tokenizer vocabulary. "
                "Only characters present in the initial corpus can be encoded."
            )
    return tokens

decode 

decode(tokens)

Decodes a list of integer tokens back into a string of text.

参数：

名称 类型 描述 默认
tokens list[int]

A list of integer tokens to decode.

 必需

返回：

名称 类型 描述
str str

The decoded string.

引发：

类型 描述
KeyError

If the list contains token IDs not present in the tokenizer's vocabulary.
源代码位于： src/llm/tokenization/simple_tokenizer.py 
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
def decode(self, tokens: list[int]) -> str:
    """
    Decodes a list of integer tokens back into a string of text.

    Args:
        tokens (list[int]): A list of integer tokens to decode.

    Returns:
        str: The decoded string.

    Raises:
        KeyError: If the list contains token IDs not present in the
                  tokenizer's vocabulary.
    """
    if not isinstance(tokens, list):
        raise TypeError("Input tokens must be a list of integers.")
    if not all(isinstance(token, int) for token in tokens):
        raise TypeError("All items in the tokens list must be integers.")

    text_chars: list[str] = []
    for token in tokens:
        try:
            text_chars.append(self.itos[token])
        except KeyError:
            raise KeyError(
                f"Token ID '{token}' not found in tokenizer vocabulary. "
                "Only token IDs derived from the initial corpus can be decoded."
            )
    return "".join(text_chars)

BPE Tokenizer

bpe_tokenizer

BPETokenizer

A Byte Pair Encoding (BPE) tokenizer using the tokenizers library.

源代码位于： src/llm/tokenization/bpe_tokenizer.py 
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
class BPETokenizer:
    """
    A Byte Pair Encoding (BPE) tokenizer using the `tokenizers` library.
    """

    def __init__(self, tokenizer: Tokenizer | None = None):
        """
        Initializes the BPETokenizer.

        Args:
            tokenizer (Tokenizer, optional): An existing tokenizers.Tokenizer instance.
        """
        if tokenizer is not None:
            self.tokenizer = tokenizer
        else:
            self.tokenizer = Tokenizer(models.BPE(unk_token=DEFAULT_UNK_TOKEN))
            self.tokenizer.normalizer = normalizers.Sequence([normalizers.NFC(), normalizers.Lowercase()])
            self.tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
            self.tokenizer.decoder = decoders.ByteLevel()

    @classmethod
    def train(
        cls,
        files: list[str],
        vocab_size: int = 5000,
        min_frequency: int = 2,
        special_tokens: list[str] | None = None,
    ) -> BPETokenizer:
        """
        Trains a BPE tokenizer on the given files.

        Args:
            files (list[str]): List of paths to text files for training.
            vocab_size (int): The desired vocabulary size.
            min_frequency (int): The minimum frequency for a pair to be merged.
            special_tokens (list[str]): List of special tokens to include.

        Returns:
            BPETokenizer: A trained tokenizer instance.
        """
        if special_tokens is None:
            special_tokens = [DEFAULT_UNK_TOKEN, "[CLS]", "[SEP]", "[PAD]", "[MASK]"]

        tokenizer = Tokenizer(models.BPE(unk_token=DEFAULT_UNK_TOKEN))
        tokenizer.normalizer = normalizers.Sequence([normalizers.NFC(), normalizers.Lowercase()])
        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
        tokenizer.decoder = decoders.ByteLevel()

        trainer = trainers.BpeTrainer(
            vocab_size=vocab_size,
            min_frequency=min_frequency,
            special_tokens=special_tokens,
            initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
        )

        tokenizer.train(files, trainer)
        return cls(tokenizer)

    def encode(self, text: str) -> list[int]:
        """
        Encodes a string into a list of token IDs.

        Args:
            text (str): The input text.

        Returns:
            list[int]: The list of token IDs.
        """
        return self.tokenizer.encode(text).ids

    def decode(self, ids: list[int], skip_special_tokens: bool = True) -> str:
        """
        Decodes a list of token IDs back into a string.

        Args:
            ids (list[int]): The list of token IDs.
            skip_special_tokens (bool): Whether to skip special tokens in the output.

        Returns:
            str: The decoded string.
        """
        return self.tokenizer.decode(ids, skip_special_tokens=skip_special_tokens)

    def save(self, path: str) -> None:
        """
        Saves the tokenizer to a file.

        Args:
            path (str): The path to save the tokenizer to.
        """
        p = Path(path)
        if not p.parent.exists():
            p.parent.mkdir(parents=True)
        self.tokenizer.save(str(p))

    @classmethod
    def load(cls, path: str) -> BPETokenizer:
        """
        Loads a tokenizer from a file.

        Args:
            path (str): The path to the tokenizer file.

        Returns:
            BPETokenizer: The loaded tokenizer instance.
        """
        return cls(Tokenizer.from_file(path))

    @property
    def vocab_size(self) -> int:
        """
        Returns the vocabulary size.
        """
        return self.tokenizer.get_vocab_size()

    def get_vocab(self) -> dict:
        """
        Returns the vocabulary mapping.
        """
        return self.tokenizer.get_vocab()

    @property
    def pad_token_id(self) -> int:
        """
        Returns the ID of the [PAD] token.

        ``token_to_id("[PAD]")`` is ``None`` whenever the trained/loaded
        vocab lacks ``[PAD]`` (custom ``special_tokens``, or a foreign
        ``tokenizer.json``). Callers fall back with ``getattr(tokenizer,
        "pad_token_id", 0)`` — but the attribute *exists* as None, so the
        default never fires and padding builds ``[None] * n``, crashing
        ``torch.tensor`` in ``batch_generate`` / ``TextDataset`` (RIL
        ISS-155). Fall back to the UNK token's id then the documented ``0``,
        so the property never returns ``None`` and never collides with
        content.

        Both lookups use explicit ``is not None`` checks, NOT an ``or``
        chain: ``[PAD]`` (or ``<unk>``) is legitimately vocab id ``0`` in
        some tokenizers, and ``0 or ...`` falls through to the WRONG token —
        padding every sample with the real content token ``<unk>`` instead of
        ``[PAD]`` (silent data corruption; deep-dive finding).
        """
        pad_id = self.tokenizer.token_to_id("[PAD]")
        if pad_id is not None:
            return pad_id
        unk_id = self.tokenizer.token_to_id(DEFAULT_UNK_TOKEN)
        return unk_id if unk_id is not None else 0

vocab_size property 

vocab_size

Returns the vocabulary size.

pad_token_id property 

pad_token_id

Returns the ID of the [PAD] token.

token_to_id("[PAD]") is None whenever the trained/loaded vocab lacks [PAD] (custom special_tokens, or a foreign tokenizer.json). Callers fall back with getattr(tokenizer, "pad_token_id", 0) — but the attribute exists as None, so the default never fires and padding builds [None] * n, crashing torch.tensor in batch_generate / TextDataset (RIL ISS-155). Fall back to the UNK token's id then the documented 0, so the property never returns None and never collides with content.

Both lookups use explicit is not None checks, NOT an or chain: [PAD] (or <unk>) is legitimately vocab id 0 in some tokenizers, and 0 or ... falls through to the WRONG token — padding every sample with the real content token <unk> instead of [PAD] (silent data corruption; deep-dive finding).

train classmethod 

train(files, vocab_size=5000, min_frequency=2, special_tokens=None)

Trains a BPE tokenizer on the given files.

参数：

名称 类型 描述 默认
files list[str]

List of paths to text files for training.

 必需
vocab_size int

The desired vocabulary size.

 5000
min_frequency int

The minimum frequency for a pair to be merged.

 2
special_tokens list[str]

List of special tokens to include.

 None

返回：

名称 类型 描述
BPETokenizer BPETokenizer

A trained tokenizer instance.
源代码位于： src/llm/tokenization/bpe_tokenizer.py 
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
@classmethod
def train(
    cls,
    files: list[str],
    vocab_size: int = 5000,
    min_frequency: int = 2,
    special_tokens: list[str] | None = None,
) -> BPETokenizer:
    """
    Trains a BPE tokenizer on the given files.

    Args:
        files (list[str]): List of paths to text files for training.
        vocab_size (int): The desired vocabulary size.
        min_frequency (int): The minimum frequency for a pair to be merged.
        special_tokens (list[str]): List of special tokens to include.

    Returns:
        BPETokenizer: A trained tokenizer instance.
    """
    if special_tokens is None:
        special_tokens = [DEFAULT_UNK_TOKEN, "[CLS]", "[SEP]", "[PAD]", "[MASK]"]

    tokenizer = Tokenizer(models.BPE(unk_token=DEFAULT_UNK_TOKEN))
    tokenizer.normalizer = normalizers.Sequence([normalizers.NFC(), normalizers.Lowercase()])
    tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
    tokenizer.decoder = decoders.ByteLevel()

    trainer = trainers.BpeTrainer(
        vocab_size=vocab_size,
        min_frequency=min_frequency,
        special_tokens=special_tokens,
        initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
    )

    tokenizer.train(files, trainer)
    return cls(tokenizer)

encode 

encode(text)

Encodes a string into a list of token IDs.

参数：

名称 类型 描述 默认
text str

The input text.

 必需

返回：

类型 描述
list[int]

list[int]: The list of token IDs.
源代码位于： src/llm/tokenization/bpe_tokenizer.py 
66
67
68
69
70
71
72
73
74
75
76
def encode(self, text: str) -> list[int]:
    """
    Encodes a string into a list of token IDs.

    Args:
        text (str): The input text.

    Returns:
        list[int]: The list of token IDs.
    """
    return self.tokenizer.encode(text).ids

decode 

decode(ids, skip_special_tokens=True)

Decodes a list of token IDs back into a string.

参数：

名称 类型 描述 默认
ids list[int]

The list of token IDs.

 必需
skip_special_tokens bool

Whether to skip special tokens in the output.

 True

返回：

名称 类型 描述
str str

The decoded string.
源代码位于： src/llm/tokenization/bpe_tokenizer.py 
78
79
80
81
82
83
84
85
86
87
88
89
def decode(self, ids: list[int], skip_special_tokens: bool = True) -> str:
    """
    Decodes a list of token IDs back into a string.

    Args:
        ids (list[int]): The list of token IDs.
        skip_special_tokens (bool): Whether to skip special tokens in the output.

    Returns:
        str: The decoded string.
    """
    return self.tokenizer.decode(ids, skip_special_tokens=skip_special_tokens)

save 

save(path)

Saves the tokenizer to a file.

参数：

名称 类型 描述 默认
path str

The path to save the tokenizer to.

 必需
源代码位于： src/llm/tokenization/bpe_tokenizer.py 
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
def save(self, path: str) -> None:
    """
    Saves the tokenizer to a file.

    Args:
        path (str): The path to save the tokenizer to.
    """
    p = Path(path)
    if not p.parent.exists():
        p.parent.mkdir(parents=True)
    self.tokenizer.save(str(p))

load classmethod 

load(path)

Loads a tokenizer from a file.

参数：

名称 类型 描述 默认
path str

The path to the tokenizer file.

 必需

返回：

名称 类型 描述
BPETokenizer BPETokenizer

The loaded tokenizer instance.
源代码位于： src/llm/tokenization/bpe_tokenizer.py 
103
104
105
106
107
108
109
110
111
112
113
114
@classmethod
def load(cls, path: str) -> BPETokenizer:
    """
    Loads a tokenizer from a file.

    Args:
        path (str): The path to the tokenizer file.

    Returns:
        BPETokenizer: The loaded tokenizer instance.
    """
    return cls(Tokenizer.from_file(path))

get_vocab 

get_vocab()

Returns the vocabulary mapping.

源代码位于： src/llm/tokenization/bpe_tokenizer.py 
123
124
125
126
127
def get_vocab(self) -> dict:
    """
    Returns the vocabulary mapping.
    """
    return self.tokenizer.get_vocab()

BPE Training

train_bpe