Class: ChunkerRuby::Character

Inherits:
BaseSplitter show all
Defined in:
lib/chunker_ruby/character.rb

Instance Attribute Summary

Attributes inherited from BaseSplitter

#chunk_overlap, #chunk_size

Instance Method Summary collapse

Methods inherited from BaseSplitter

#initialize, #split_many

Constructor Details

This class inherits a constructor from ChunkerRuby::BaseSplitter

Instance Method Details

#split(text, metadata: {}) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/chunker_ruby/character.rb', line 5

def split(text, metadata: {})
  return [] if text.nil? || text.empty?

  chunks = []
  start = 0

  while start < text.length
    end_pos = [start + @chunk_size, text.length].min
    chunk_text = text[start...end_pos]

    chunks << Chunk.new(
      text: chunk_text,
      index: chunks.size,
      offset: start,
      metadata: metadata.dup
    )

    break if end_pos >= text.length

    start += @chunk_size - @chunk_overlap
  end

  chunks
end