Method List
-
#char_to_token Tokenizers::Encoding
-
#char_to_word Tokenizers::Encoding
-
#decode Tokenizers::CharBPETokenizer
-
#encode Tokenizers::Tokenizer
-
#encode Tokenizers::CharBPETokenizer
-
#from_pretrained Tokenizers::FromPretrained
-
#initialize Tokenizers::CharBPETokenizer
-
#word_to_chars Tokenizers::Encoding
-
#word_to_tokens Tokenizers::Encoding