Class: ChunkerRuby::JSONSplitter

Inherits:
BaseSplitter show all
Defined in:
lib/chunker_ruby/json_splitter.rb

Instance Attribute Summary

Attributes inherited from BaseSplitter

#chunk_overlap, #chunk_size

Instance Method Summary collapse

Methods inherited from BaseSplitter

#initialize, #split_many

Constructor Details

This class inherits a constructor from ChunkerRuby::BaseSplitter

Instance Method Details

#split(text, metadata: {}) ⇒ Object 



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/chunker_ruby/json_splitter.rb', line 7

def split(text, metadata: {})
  return [] if text.nil? || text.empty?

  parsed = ::JSON.parse(text)
  pieces = extract_pieces(parsed)
  chunks = []
  current_pos = 0

  current_parts = []
  current_length = 0

  pieces.each do |piece|
    json_str = ::JSON.generate(piece)

    if current_length + json_str.length > @chunk_size && !current_parts.empty?
      chunk_text = ::JSON.generate(current_parts.length == 1 ? current_parts.first : current_parts)
      # Search for a key or value from the first piece to approximate offset
      offset = find_json_offset(text, current_parts.first, current_pos)
      current_pos = offset + chunk_text.length
      chunks << Chunk.new(
        text: chunk_text,
        index: chunks.size,
        offset: offset,
        metadata: metadata.dup
      )
      current_parts = []
      current_length = 0
    end

    current_parts << piece
    current_length += json_str.length
  end

  unless current_parts.empty?
    chunk_text = ::JSON.generate(current_parts.length == 1 ? current_parts.first : current_parts)
    offset = find_json_offset(text, current_parts.first, current_pos)
    chunks << Chunk.new(
      text: chunk_text,
      index: chunks.size,
      offset: offset,
      metadata: metadata.dup
    )
  end

  chunks
end