Class: NanoGPT::BPETextfilePreparer

Inherits:
Object
  • Object
show all
Defined in:
lib/nano_gpt/bpe_textfile_preparer.rb

Overview

Prepares custom text files for training with GPT-2 BPE tokenization Mirrors TextfilePreparer but uses tiktoken instead of character-level encoding. Does NOT write meta.json -- absence triggers GPT-2 auto-detection in Tokenizer.for_dataset.

Constant Summary collapse

BUFFER_SIZE = 
100_000

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(input_path:, output_name: nil, val_ratio: 0.1) ⇒ BPETextfilePreparer

Returns a new instance of BPETextfilePreparer.



15
16
17
18
19
20
 
# File 'lib/nano_gpt/bpe_textfile_preparer.rb', line 15

def initialize(input_path:, output_name: nil, val_ratio: 0.1)
  @input_path = input_path
  @val_ratio = val_ratio
  @output_name = output_name || derive_output_name(input_path)
  @output_dir = File.join(Dir.pwd, "data", @output_name)
end

Instance Attribute Details

#input_path ⇒ Object (readonly)

Returns the value of attribute input_path.



13
14
15
 
# File 'lib/nano_gpt/bpe_textfile_preparer.rb', line 13

def input_path
  @input_path
end

#output_dir ⇒ Object (readonly)

Returns the value of attribute output_dir.



13
14
15
 
# File 'lib/nano_gpt/bpe_textfile_preparer.rb', line 13

def output_dir
  @output_dir
end

#val_ratio ⇒ Object (readonly)

Returns the value of attribute val_ratio.



13
14
15
 
# File 'lib/nano_gpt/bpe_textfile_preparer.rb', line 13

def val_ratio
  @val_ratio
end

Instance Method Details

#prepare ⇒ Object 



22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/nano_gpt/bpe_textfile_preparer.rb', line 22

def prepare
  validate_input!
  FileUtils.mkdir_p(@output_dir)

  print_header
  encoding = detect_encoding
  tokens = tokenize_file(encoding)
  train_tokens, val_tokens = split_tokens(tokens)

  write_bin(File.join(@output_dir, "train.bin"), train_tokens, "train")
  write_bin(File.join(@output_dir, "val.bin"), val_tokens, "val")

  print_summary(train_tokens.size, val_tokens.size)
  @output_name
end