Class: ChunkerRuby::Semantic

Inherits:
BaseSplitter show all
Defined in:
lib/chunker_ruby/semantic.rb

Instance Attribute Summary

Attributes inherited from BaseSplitter

#chunk_overlap, #chunk_size

Instance Method Summary collapse

Methods inherited from BaseSplitter

#split_many

Constructor Details

#initialize(embed:, threshold: 0.5, min_chunk_size: 100, max_chunk_size: 2000, **kwargs) ⇒ Semantic

Returns a new instance of Semantic.



5
6
7
8
9
10
 
# File 'lib/chunker_ruby/semantic.rb', line 5

def initialize(embed:, threshold: 0.5, min_chunk_size: 100, max_chunk_size: 2000, **kwargs)
  super(chunk_size: max_chunk_size, chunk_overlap: 0, **kwargs)
  @embed = embed
  @threshold = threshold
  @min_chunk_size = min_chunk_size
end

Instance Method Details

#split(text, metadata: {}) ⇒ Object 



12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/chunker_ruby/semantic.rb', line 12

def split(text, metadata: {})
  return [] if text.nil? || text.empty?

  sentences = split_into_sentences(text)
  return [Chunk.new(text: text, index: 0, offset: 0, metadata: metadata)] if sentences.length <= 1

  embeddings = sentences.map { |s| @embed.call(s) }
  split_points = find_split_points(embeddings)

  build_semantic_chunks(sentences, split_points, text, metadata)
end