Class: LangsmithrbRails::Generators::EvalsGenerator

Inherits:
Rails::Generators::Base
  • Object
show all
Defined in:
lib/generators/langsmithrb_rails/evals/evals_generator.rb

Overview

Generator for adding LangSmith evaluations to Rails applications

Instance Method Summary collapse

Instance Method Details

#create_directories ⇒ Object 



11
12
13
14
15
 
# File 'lib/generators/langsmithrb_rails/evals/evals_generator.rb', line 11

def create_directories
  empty_directory "config/langsmith/evals/datasets"
  empty_directory "config/langsmith/evals/checks"
  empty_directory "config/langsmith/evals/targets"
end

#create_rake_task ⇒ Object 



31
32
33
 
# File 'lib/generators/langsmithrb_rails/evals/evals_generator.rb', line 31

def create_rake_task
  template "langsmith_evals.rake", "lib/tasks/langsmith_evals.rake"
end

#create_sample_check ⇒ Object 



21
22
23
24
 
# File 'lib/generators/langsmithrb_rails/evals/evals_generator.rb', line 21

def create_sample_check
  template "checks/correctness.rb", "config/langsmith/evals/checks/correctness.rb"
  template "checks/llm_graded.rb", "config/langsmith/evals/checks/llm_graded.rb"
end

#create_sample_dataset ⇒ Object 



17
18
19
 
# File 'lib/generators/langsmithrb_rails/evals/evals_generator.rb', line 17

def create_sample_dataset
  template "datasets/sample.yml", "config/langsmith/evals/datasets/sample.yml"
end

#create_sample_target ⇒ Object 



26
27
28
29
 
# File 'lib/generators/langsmithrb_rails/evals/evals_generator.rb', line 26

def create_sample_target
  template "targets/http.rb", "config/langsmith/evals/targets/http.rb"
  template "targets/ruby.rb", "config/langsmith/evals/targets/ruby.rb"
end

#display_post_install_message ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/generators/langsmithrb_rails/evals/evals_generator.rb', line 35

def display_post_install_message
  say "\n"
  say "LangSmith evaluations have been added to your Rails application! 🎉", :green
  say "\n"
  say "Usage:", :yellow
  say "  1. Customize the sample dataset in config/langsmith/evals/datasets/", :yellow
  say "  2. Run an evaluation:", :yellow
  say "     bin/rails langsmith:eval[sample,http,my_experiment]", :yellow
  say "  3. Compare experiments:", :yellow
  say "     bin/rails langsmith:compare[exp_a,exp_b]", :yellow
  say "\n"
  say "To add CI integration for evaluations, run:", :yellow
  say "  bin/rails g langsmithrb_rails:ci", :yellow
  say "\n"
end