Class: Tiktoken::Encoder

Inherits:
Object
  • Object
show all
Defined in:
lib/tiktoken/encoder.rb

Instance Method Summary collapse

Instance Method Details

#decode(tokens, utf_opts = {invalid: :replace, undef: :replace}) ⇒ Object 



11
12
13
14
 
# File 'lib/tiktoken/encoder.rb', line 11

def decode(tokens, utf_opts={invalid: :replace, undef: :replace})
    _bytes = _decode(tokens)
    _bytes.pack('C*').encode('UTF-8', **utf_opts)
end

#encode(text, allowed_special = [], disallowed_special = "all") ⇒ Object 



3
4
5
 
# File 'lib/tiktoken/encoder.rb', line 3

def encode(text, allowed_special=[], disallowed_special="all")
    _encode(text, allowed_special, disallowed_special)
end

#encode_ordinary(text) ⇒ Object 



7
8
9
 
# File 'lib/tiktoken/encoder.rb', line 7

def encode_ordinary(text)
    _encode_ordinary(text)
end