11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
|
# File 'lib/chunker_ruby/rails/chunkable.rb', line 11
def chunkable(attribute, strategy: :recursive_character, chunk_size: 1000, chunk_overlap: 200, **options)
@chunkable_config = {
attribute: attribute,
strategy: strategy,
chunk_size: chunk_size,
chunk_overlap: chunk_overlap,
options: options
}
after_save :rechunk!, if: -> { saved_change_to_attribute?(attribute) }
has_many :chunks,
class_name: "#{name}Chunk",
dependent: :destroy
define_method(:chunker) do
config = self.class.instance_variable_get(:@chunkable_config)
splitter_class = ChunkerRuby::Rails::Chunkable.resolve_strategy(config[:strategy])
splitter_class.new(
chunk_size: config[:chunk_size],
chunk_overlap: config[:chunk_overlap],
**config[:options]
)
end
define_method(:rechunk!) do
config = self.class.instance_variable_get(:@chunkable_config)
content = send(config[:attribute])
return if content.nil? || content.empty?
chunks.destroy_all
result = chunker.split(content, metadata: { source_id: id, source_type: self.class.name })
result.each do |chunk|
chunks.create!(
text: chunk.text,
chunk_index: chunk.index,
offset: chunk.offset,
metadata: chunk.metadata
)
end
end
end
|