Class: NanoGPT::TrainConfig

Inherits:
BaseConfig show all
Defined in:
lib/nano_gpt/train_config.rb

Overview

Configuration for training

Constant Summary collapse

DEFAULTS = 
{
  # I/O
  out_dir: "out-shakespeare-char",
  eval_interval: 250,
  log_interval: 10,
  eval_iters: 200,
  eval_only: false,
  always_save_checkpoint: false,
  init_from: "scratch",

  # Data
  dataset: "shakespeare_char",
  batch_size: 64,
  block_size: 256,
  gradient_accumulation_steps: 1,

  # Model
  n_layer: 6,
  n_head: 6,
  n_embd: 384,
  dropout: 0.2,
  bias: false,

  # Optimizer
  learning_rate: 1e-3,
  weight_decay: 1e-1,
  beta1: 0.9,
  beta2: 0.99,
  grad_clip: 1.0,

  # LR scheduler
  decay_lr: true,
  warmup_iters: 100,
  lr_decay_iters: 5000,
  min_lr: 1e-4,

  # Training
  max_iters: 5000,

  # System
  device: "auto"
}.freeze

Instance Attribute Summary

Attributes inherited from BaseConfig

#values

Class Method Summary collapse

Methods inherited from BaseConfig

#[], #[]=, #initialize, load, #load_json, #save_json, #to_h

Constructor Details

This class inherits a constructor from NanoGPT::BaseConfig

Class Method Details

.defaults ⇒ Object 



149
150
151
 
# File 'lib/nano_gpt/train_config.rb', line 149

def self.defaults
  DEFAULTS
end