Skip to content

Commit 74ac182

Browse files
fix: sync text_signature according to real default values
1 parent 4bc8be5 commit 74ac182

File tree

2 files changed

+4
-3
lines changed

2 files changed

+4
-3
lines changed

bindings/python/py_src/tokenizers/trainers/__init__.pyi

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ class UnigramTrainer(Trainer):
9999
vocab_size=8000,
100100
show_progress=True,
101101
special_tokens=[],
102+
initial_alphabet=[],
102103
shrinking_factor=0.75,
103104
unk_token=None,
104105
max_piece_length=16,
@@ -123,7 +124,7 @@ class WordLevelTrainer(Trainer):
123124
special_tokens (:obj:`List[Union[str, AddedToken]]`):
124125
A list of special tokens the model should know of.
125126
"""
126-
def __init__(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=None):
127+
def __init__(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=[]):
127128
pass
128129

129130
class WordPieceTrainer(Trainer):

bindings/python/src/trainers.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -664,7 +664,7 @@ impl PyWordLevelTrainer {
664664
#[new]
665665
#[pyo3(
666666
signature = (**kwargs),
667-
text_signature = "(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=None)"
667+
text_signature = "(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=[])"
668668
)]
669669
pub fn new(kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<(Self, PyTrainer)> {
670670
let mut builder = tk::models::wordlevel::WordLevelTrainer::builder();
@@ -832,7 +832,7 @@ impl PyUnigramTrainer {
832832
#[new]
833833
#[pyo3(
834834
signature = (**kwargs),
835-
text_signature = "(self, vocab_size=8000, show_progress=True, special_tokens=[], shrinking_factor=0.75, unk_token=None, max_piece_length=16, n_sub_iterations=2)"
835+
text_signature = "(self, vocab_size=8000, show_progress=True, special_tokens=[], initial_alphabet=[], shrinking_factor=0.75, unk_token=None, max_piece_length=16, n_sub_iterations=2)"
836836
)]
837837
pub fn new(kwargs: Option<Bound<'_, PyDict>>) -> PyResult<(Self, PyTrainer)> {
838838
let mut builder = tk::models::unigram::UnigramTrainer::builder();

0 commit comments

Comments
 (0)