Module: Legion::Extensions::Knowledge::Runners::Ingest

Included in:
Client
Defined in:
lib/legion/extensions/knowledge/runners/ingest.rb

Class Method Summary collapse

Class Method Details

.ingest_corpus(path:, dry_run: false, force: false) ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/legion/extensions/knowledge/runners/ingest.rb', line 25

def ingest_corpus(path:, dry_run: false, force: false)
  entries = Helpers::Manifest.scan(path: path)

  files_scanned   = entries.size
  chunks_created  = 0
  chunks_skipped  = 0
  chunks_updated  = 0

  entries.each do |entry|
    result = process_file(entry[:path], dry_run: dry_run, force: force)
    chunks_created  += result[:created]
    chunks_skipped  += result[:skipped]
    chunks_updated  += result[:updated]
  end

  {
    success:        true,
    files_scanned:  files_scanned,
    chunks_created: chunks_created,
    chunks_skipped: chunks_skipped,
    chunks_updated: chunks_updated
  }
rescue StandardError => e
  { success: false, error: e.message }
end

.ingest_file(file_path:, force: false) ⇒ Object 



51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/legion/extensions/knowledge/runners/ingest.rb', line 51

def ingest_file(file_path:, force: false)
  result = process_file(file_path, dry_run: false, force: force)

  {
    success:        true,
    file:           file_path,
    chunks_created: result[:created],
    chunks_skipped: result[:skipped],
    chunks_updated: result[:updated]
  }
rescue StandardError => e
  { success: false, error: e.message }
end

.scan_corpus(path:, extensions: nil) ⇒ Object 



10
11
12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/legion/extensions/knowledge/runners/ingest.rb', line 10

def scan_corpus(path:, extensions: nil)
  opts = { path: path }
  opts[:extensions] = extensions if extensions

  entries = Helpers::Manifest.scan(**opts)

  {
    success:     true,
    path:        path,
    file_count:  entries.size,
    total_bytes: entries.sum { |e| e[:size] },
    files:       entries.map { |e| e[:path] }
  }
end