Class: ChunkerRuby::RecursiveCharacter

Inherits:
BaseSplitter show all
Defined in:
lib/chunker_ruby/recursive_character.rb

Constant Summary collapse

DEFAULT_SEPARATORS = 
["\n\n", "\n", ". ", ", ", " ", ""].freeze

Instance Attribute Summary

Attributes inherited from BaseSplitter

#chunk_overlap, #chunk_size

Instance Method Summary collapse

Methods inherited from BaseSplitter

#split_many

Constructor Details

#initialize(separators: nil, keep_separator: true, **kwargs) ⇒ RecursiveCharacter

Returns a new instance of RecursiveCharacter.



7
8
9
10
11
 
# File 'lib/chunker_ruby/recursive_character.rb', line 7

def initialize(separators: nil, keep_separator: true, **kwargs)
  super(**kwargs)
  @separators = separators || DEFAULT_SEPARATORS
  @keep_separator = keep_separator
end

Instance Method Details

#split(text, metadata: {}) ⇒ Object 



13
14
15
16
17
18
 
# File 'lib/chunker_ruby/recursive_character.rb', line 13

def split(text, metadata: {})
  return [] if text.nil? || text.empty?

  chunks = recursive_split(text, @separators)
  merge_chunks(chunks, text, metadata: metadata)
end