Skip to content

Commit 4bc8be5

Browse files
fix: default value
1 parent 7d556df commit 4bc8be5

File tree

2 files changed

+17
-1
lines changed

2 files changed

+17
-1
lines changed

bindings/python/py_src/tokenizers/trainers/__init__.pyi

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,20 @@ class BpeTrainer(Trainer):
4545
highly repetitive tokens like `======` for wikipedia
4646
4747
"""
48+
def __init__(
49+
self,
50+
vocab_size=30000,
51+
min_frequency=0,
52+
show_progress=True,
53+
special_tokens=[],
54+
limit_alphabet=None,
55+
initial_alphabet=[],
56+
continuing_subword_prefix=None,
57+
end_of_word_suffix=None,
58+
max_token_length=None,
59+
words={},
60+
):
61+
pass
4862

4963
class UnigramTrainer(Trainer):
5064
"""
@@ -109,6 +123,8 @@ class WordLevelTrainer(Trainer):
109123
special_tokens (:obj:`List[Union[str, AddedToken]]`):
110124
A list of special tokens the model should know of.
111125
"""
126+
def __init__(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=None):
127+
pass
112128

113129
class WordPieceTrainer(Trainer):
114130
"""

bindings/python/src/trainers.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -664,7 +664,7 @@ impl PyWordLevelTrainer {
664664
#[new]
665665
#[pyo3(
666666
signature = (**kwargs),
667-
text_signature = "(self, vocab_size=3000, min_frequency=0, show_progress=True, special_tokens=[])"
667+
text_signature = "(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=None)"
668668
)]
669669
pub fn new(kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<(Self, PyTrainer)> {
670670
let mut builder = tk::models::wordlevel::WordLevelTrainer::builder();

0 commit comments

Comments
 (0)