Class: Tiktoken::Encoding

Inherits:
Object
  • Object
show all
Defined in:
lib/tiktoken_ruby/encoding.rb

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(ext_base_bpe) ⇒ Encoding

Returns a new instance of Encoding.



8
9
10
 
# File 'lib/tiktoken_ruby/encoding.rb', line 8

def initialize(ext_base_bpe)
    @ext_base_bpe = ext_base_bpe
end

Class Method Details

.method_missing(method) ⇒ Object 



4
5
6
 
# File 'lib/tiktoken_ruby/encoding.rb', line 4

def self.method_missing(method)
    Tiktoken::Encoding.new(Tiktoken::BpeFactory.send(method))
end

Instance Method Details

#decode(tokens) ⇒ Object 



20
21
22
 
# File 'lib/tiktoken_ruby/encoding.rb', line 20

def decode(tokens)
    @ext_base_bpe.decode(tokens)
end

#encode(text, allowed_special: []) ⇒ Object 



16
17
18
 
# File 'lib/tiktoken_ruby/encoding.rb', line 16

def encode(text, allowed_special: [])
    @ext_base_bpe.encode(text, allowed_special)
end

#encode_ordinary(text) ⇒ Object 



12
13
14
 
# File 'lib/tiktoken_ruby/encoding.rb', line 12

def encode_ordinary(text)
    @ext_base_bpe.encode_ordinary(text)
end