Class: RagRuby::Pipeline

Inherits:
Object
  • Object
show all
Defined in:
lib/rag_ruby/pipeline.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize {|@config| ... } ⇒ Pipeline

Returns a new instance of Pipeline.

Yields:



9
10
11
12
13
14
 
# File 'lib/rag_ruby/pipeline.rb', line 9

def initialize(&block)
  @config = Configuration.new
  yield @config if block_given?
  @prompt_template = PromptTemplate.new
  @chunk_store = {} # id -> chunk mapping
end

Instance Attribute Details

#config ⇒ Object (readonly)

Returns the value of attribute config.



7
8
9
 
# File 'lib/rag_ruby/pipeline.rb', line 7

def config
  @config
end

Instance Method Details

#ingest(source, loader: nil) ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/rag_ruby/pipeline.rb', line 16

def ingest(source, loader: nil)
  loader ||= @config.loader_instance || Loaders::File.new

  # Load documents
  fire(:before_load, source)
  documents = loader.load(source)
  fire(:after_load, documents)

  documents.each do |doc|
    next if doc.empty?

    # Chunk
    fire(:before_chunk, doc)
    chunks = chunk_document(doc)
    fire(:after_chunk, chunks)

    # Embed
    fire(:before_embed, chunks)
    embed_chunks(chunks)
    fire(:after_embed, chunks)

    # Store
    fire(:before_store, chunks)
    store_chunks(chunks)
    fire(:after_store, chunks)
  end
end

#ingest_directory(dir_path, glob: "**/*.{md,txt}") ⇒ Object 



44
45
46
47
 
# File 'lib/rag_ruby/pipeline.rb', line 44

def ingest_directory(dir_path, glob: "**/*.{md,txt}")
  loader = Loaders::Directory.new(glob: glob)
  ingest(dir_path, loader: loader)
end

#query(question, top_k: 5, filter: nil, temperature: 0.7, system_prompt: nil) ⇒ Object 



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/rag_ruby/pipeline.rb', line 49

def query(question, top_k: 5, filter: nil, temperature: 0.7, system_prompt: nil)
  start_time = Time.now

  fire(:before_query, question)

  # Embed the question
  query_embedding = @config.embedder_instance.embed(question)

  # Search the store (with retrieval strategy)
  search_opts = { top_k: top_k, filter: filter }
  if @config.retrieval_strategy == :mmr
    search_opts[:strategy] = :mmr
    search_opts[:lambda] = @config.mmr_lambda
    search_opts[:fetch_k] = @config.mmr_fetch_k
  end

  results = @config.store_instance.search(query_embedding, **search_opts)

  # Rerank if configured
  if @config.reranker_instance
    results = rerank_results(question, results)
  end

  # Build sources from results
  sources = results.map do |result|
    chunk = result[:chunk] || @chunk_store[result[:id]]
    next unless chunk

    Source.new(chunk: chunk, score: result[:score])
  end.compact

  # Build context from retrieved chunks
  context = build_context(sources)

  # Generate answer
  prompt_text = @prompt_template.render(context: context, question: question)
  sys_prompt = system_prompt || @prompt_template.system_prompt

  gen_result = @config.generator_instance.generate(
    prompt: prompt_text,
    system_prompt: sys_prompt,
    temperature: temperature
  )

  duration = Time.now - start_time

  answer = Answer.new(
    text: gen_result[:text],
    sources: sources,
    tokens_used: gen_result[:tokens_used],
    duration: duration,
    query: question
  )

  fire(:after_query, question, answer)

  answer
end