Class: ChunkerRuby::HTML

Inherits:
BaseSplitter show all
Defined in:
lib/chunker_ruby/html.rb

Constant Summary collapse

BLOCK_TAGS = 
%w[
  div p section article aside main header footer nav
  h1 h2 h3 h4 h5 h6 blockquote pre ul ol li table tr
  form fieldset details summary figure figcaption
].freeze

Instance Attribute Summary

Attributes inherited from BaseSplitter

#chunk_overlap, #chunk_size

Instance Method Summary collapse

Methods inherited from BaseSplitter

#split_many

Constructor Details

#initialize(strip_tags: false, **kwargs) ⇒ HTML

Returns a new instance of HTML.



11
12
13
14
 
# File 'lib/chunker_ruby/html.rb', line 11

def initialize(strip_tags: false, **kwargs)
  super(**kwargs)
  @strip_tags = strip_tags
end

Instance Method Details

#split(text, metadata: {}) ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/chunker_ruby/html.rb', line 16

def split(text, metadata: {})
  return [] if text.nil? || text.empty?

  sections = split_by_tags(text)
  chunks = []

  sections.each do |section|
    content = @strip_tags ? strip_html_tags(section[:text]) : section[:text]
    next if content.strip.empty?

    if content.length <= @chunk_size
      chunks << Chunk.new(
        text: content,
        index: chunks.size,
        offset: section[:offset],
        metadata: metadata.dup
      )
    else
      sub_splitter = RecursiveCharacter.new(
        chunk_size: @chunk_size,
        chunk_overlap: @chunk_overlap
      )
      sub_chunks = sub_splitter.split(content, metadata: metadata)
      sub_chunks.each do |sc|
        chunks << Chunk.new(
          text: sc.text,
          index: chunks.size,
          offset: section[:offset] + sc.offset,
          metadata: sc.metadata
        )
      end
    end
  end

  chunks
end