Class: ChunkerRuby::Sentence

Inherits:
BaseSplitter show all
Defined in:
lib/chunker_ruby/sentence.rb

Constant Summary collapse

ABBREVIATIONS = 
%w[
  Mr Mrs Ms Dr Prof Sr Jr St Gen Gov Sgt Cpl Pvt
  Inc Corp Ltd Co vs etc al
  Jan Feb Mar Apr Jun Jul Aug Sep Oct Nov Dec
  Ave Blvd Dept Div Est Fig
].freeze

Instance Attribute Summary

Attributes inherited from BaseSplitter

#chunk_overlap, #chunk_size

Instance Method Summary collapse

Methods inherited from BaseSplitter

#split_many

Constructor Details

#initialize(min_chunk_size: nil, max_chunk_size: nil, **kwargs) ⇒ Sentence

Returns a new instance of Sentence.



14
15
16
17
18
 
# File 'lib/chunker_ruby/sentence.rb', line 14

def initialize(min_chunk_size: nil, max_chunk_size: nil, **kwargs)
  chunk_size = max_chunk_size || kwargs[:chunk_size] || 1000
  super(chunk_size: chunk_size, **kwargs.except(:chunk_size))
  @min_chunk_size = min_chunk_size || (@chunk_size / 3)
end

Instance Method Details

#split(text, metadata: {}) ⇒ Object 



20
21
22
23
24
25
 
# File 'lib/chunker_ruby/sentence.rb', line 20

def split(text, metadata: {})
  return [] if text.nil? || text.empty?

  sentences = split_into_sentences(text)
  build_chunks(sentences, text, metadata: metadata)
end