Class: NanoGPT::TextfilePreparer

Inherits:
Object
  • Object
show all
Defined in:
lib/nano_gpt/textfile_preparer.rb

Overview

Prepares custom text files for training with character-level tokenization Handles large files efficiently through streaming

Constant Summary collapse

BUFFER_SIZE = 
100_000

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(input_path:, output_name: nil, val_ratio: 0.1) ⇒ TextfilePreparer

Returns a new instance of TextfilePreparer.



16
17
18
19
20
21
 
# File 'lib/nano_gpt/textfile_preparer.rb', line 16

def initialize(input_path:, output_name: nil, val_ratio: 0.1)
  @input_path = input_path
  @val_ratio = val_ratio
  @output_name = output_name || derive_output_name(input_path)
  @output_dir = File.join(Dir.pwd, "data", @output_name)
end

Instance Attribute Details

#input_path ⇒ Object (readonly)

Returns the value of attribute input_path.



14
15
16
 
# File 'lib/nano_gpt/textfile_preparer.rb', line 14

def input_path
  @input_path
end

#output_dir ⇒ Object (readonly)

Returns the value of attribute output_dir.



14
15
16
 
# File 'lib/nano_gpt/textfile_preparer.rb', line 14

def output_dir
  @output_dir
end

#val_ratio ⇒ Object (readonly)

Returns the value of attribute val_ratio.



14
15
16
 
# File 'lib/nano_gpt/textfile_preparer.rb', line 14

def val_ratio
  @val_ratio
end

Instance Method Details

#prepare ⇒ Object 



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/nano_gpt/textfile_preparer.rb', line 23

def prepare
  validate_input!
  FileUtils.mkdir_p(@output_dir)

  print_header
  encoding = detect_encoding
  vocab, char_count = build_vocabulary(encoding)
  stoi, itos = build_mappings(vocab)
  train_chars, val_chars = calculate_split(char_count)

  write_train_bin(encoding, stoi, train_chars)
  write_val_bin(encoding, stoi, train_chars, val_chars)
  write_meta_json(vocab.size, stoi, itos)

  print_summary(train_chars, val_chars, vocab.size)
  @output_name
end