Class: ChunkerRuby::Code

Inherits:
BaseSplitter show all
Defined in:
lib/chunker_ruby/code.rb

Constant Summary collapse

LANGUAGE_SEPARATORS = 
{
  ruby: [
    "\nclass ", "\nmodule ", "\ndef ", "\n\n", "\n", " ", ""
  ],
  python: [
    "\nclass ", "\ndef ", "\n\n", "\n", " ", ""
  ],
  javascript: [
    "\nfunction ", "\nclass ", "\nconst ", "\nlet ", "\nvar ",
    "\nexport ", "\n\n", "\n", " ", ""
  ],
  typescript: [
    "\ninterface ", "\ntype ", "\nfunction ", "\nclass ",
    "\nconst ", "\nlet ", "\nexport ", "\n\n", "\n", " ", ""
  ]
}.freeze

Instance Attribute Summary

Attributes inherited from BaseSplitter

#chunk_overlap, #chunk_size

Instance Method Summary collapse

Methods inherited from BaseSplitter

#split_many

Constructor Details

#initialize(language: :ruby, **kwargs) ⇒ Code

Returns a new instance of Code.



22
23
24
25
26
27
28
 
# File 'lib/chunker_ruby/code.rb', line 22

def initialize(language: :ruby, **kwargs)
  super(**kwargs)
  @language = language.to_sym
  @separators = LANGUAGE_SEPARATORS.fetch(@language) do
    raise ArgumentError, "Unsupported language: #{language}. Supported: #{LANGUAGE_SEPARATORS.keys.join(", ")}"
  end
end

Instance Method Details

#split(text, metadata: {}) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/chunker_ruby/code.rb', line 30

def split(text, metadata: {})
  return [] if text.nil? || text.empty?

  meta = metadata.merge(language: @language)
  splitter = RecursiveCharacter.new(
    chunk_size: @chunk_size,
    chunk_overlap: @chunk_overlap,
    separators: @separators,
    keep_separator: true
  )
  splitter.split(text, metadata: meta)
end