Class: Tokenizers::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/tokenizers/tokenizer.rb

Instance Method Summary collapse

Instance Method Details

#encode(sequence, add_special_tokens: nil) ⇒ Object

TODO change add_special_tokens default to true in 0.3.0



4
5
6
7
8
9
10
 
# File 'lib/tokenizers/tokenizer.rb', line 4

def encode(sequence, add_special_tokens: nil)
  if add_special_tokens.nil?
    warn "[tokenizers] add_special_tokens will default to true in 0.3.0. Pass add_special_tokens: true/false to silence this warning."
    add_special_tokens = false
  end
  _encode(sequence, add_special_tokens)
end