Class: ChunkerRuby::Token

Inherits:
BaseSplitter show all
Defined in:
lib/chunker_ruby/token.rb

Instance Attribute Summary

Attributes inherited from BaseSplitter

#chunk_overlap, #chunk_size

Instance Method Summary collapse

Methods inherited from BaseSplitter

#split_many

Constructor Details

#initialize(tokenizer: nil, **kwargs) ⇒ Token

Returns a new instance of Token.



5
6
7
8
 
# File 'lib/chunker_ruby/token.rb', line 5

def initialize(tokenizer: nil, **kwargs)
  super(**kwargs)
  @tokenizer = resolve_tokenizer(tokenizer)
end

Instance Method Details

#split(text, metadata: {}) ⇒ Object 



10
11
12
13
14
15
16
17
18
 
# File 'lib/chunker_ruby/token.rb', line 10

def split(text, metadata: {})
  return [] if text.nil? || text.empty?

  if @tokenizer
    split_by_tokens(text, metadata)
  else
    split_by_estimation(text, metadata)
  end
end