Class: ChunkerRuby::Chunk

Inherits:
Object
  • Object
show all
Defined in:
lib/chunker_ruby/chunk.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(text:, index:, offset:, metadata: {}) ⇒ Chunk

Returns a new instance of Chunk.



7
8
9
10
11
12
13
 
# File 'lib/chunker_ruby/chunk.rb', line 7

def initialize(text:, index:, offset:, metadata: {})
  @text = text
  @index = index
  @offset = offset
  @length = text.length
  @metadata = metadata
end

Instance Attribute Details

#index ⇒ Object (readonly)

Returns the value of attribute index.



5
6
7
 
# File 'lib/chunker_ruby/chunk.rb', line 5

def index
  @index
end

#length ⇒ Object (readonly)

Returns the value of attribute length.



5
6
7
 
# File 'lib/chunker_ruby/chunk.rb', line 5

def length
  @length
end

#metadata ⇒ Object (readonly)

Returns the value of attribute metadata.



5
6
7
 
# File 'lib/chunker_ruby/chunk.rb', line 5

def metadata
  @metadata
end

#offset ⇒ Object (readonly)

Returns the value of attribute offset.



5
6
7
 
# File 'lib/chunker_ruby/chunk.rb', line 5

def offset
  @offset
end

#text ⇒ Object (readonly)

Returns the value of attribute text.



5
6
7
 
# File 'lib/chunker_ruby/chunk.rb', line 5

def text
  @text
end

Instance Method Details

#==(other) ⇒ Object 



42
43
44
 
# File 'lib/chunker_ruby/chunk.rb', line 42

def ==(other)
  other.is_a?(Chunk) && text == other.text && index == other.index && offset == other.offset
end

#to_h ⇒ Object 



28
29
30
 
# File 'lib/chunker_ruby/chunk.rb', line 28

def to_h
  { text: @text, index: @index, offset: @offset, length: @length, metadata: @metadata }
end

#to_s ⇒ Object 



24
25
26
 
# File 'lib/chunker_ruby/chunk.rb', line 24

def to_s
  @text
end

#token_count(tokenizer = nil) ⇒ Object 



15
16
17
18
19
20
21
22
 
# File 'lib/chunker_ruby/chunk.rb', line 15

def token_count(tokenizer = nil)
  if tokenizer
    tokenizer.encode(text).length
  else
    # Rough estimation: ~4 characters per token for English
    (text.length / 4.0).ceil
  end
end

#valid?(original_text = nil) ⇒ Boolean

Returns:

  • (Boolean) 


32
33
34
35
36
37
38
39
40
 
# File 'lib/chunker_ruby/chunk.rb', line 32

def valid?(original_text = nil)
  return false if text.nil? || text.empty?
  return false if offset.negative?
  return false if index.negative?
  if original_text
    return false unless original_text[offset, text.length] == text
  end
  true
end