Class: SVMKit::ModelSelection::CrossValidation

Inherits:
Object
  • Object
show all
Defined in:
lib/svmkit/model_selection/cross_validation.rb

Overview

CrossValidation is a class that evaluates a given classifier with cross-validation method.

Examples:

svc = SVMKit::LinearModel::SVC.new
kf = SVMKit::ModelSelection::StratifiedKFold.new(n_splits: 5)
cv = SVMKit::ModelSelection::CrossValidation.new(estimator: svc, splitter: kf)
report = cv.perform(samples, lables)
mean_test_score = report[:test_score].inject(:+) / kf.n_splits

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(estimator: nil, splitter: nil, evaluator: nil, return_train_score: false) ⇒ CrossValidation

Create a new evaluator with cross-validation method.

Parameters:

  • estimator (Classifier) (defaults to: nil) —

    The classifier of which performance is evaluated.

  • splitter (Splitter) (defaults to: nil) —

    The splitter that divides dataset to training and testing dataset.

  • evaluator (Evaluator) (defaults to: nil) —

    The evaluator that calculates score of estimator results.

  • return_train_score (Boolean) (defaults to: false) —

    The flag indicating whether to calculate the score of training dataset.



46
47
48
49
50
51
52
53
54
55
 
# File 'lib/svmkit/model_selection/cross_validation.rb', line 46

def initialize(estimator: nil, splitter: nil, evaluator: nil, return_train_score: false)
  SVMKit::Validation.check_params_type(SVMKit::Base::BaseEstimator, estimator: estimator)
  SVMKit::Validation.check_params_type(SVMKit::Base::Splitter, splitter: splitter)
  SVMKit::Validation.check_params_type_or_nil(SVMKit::Base::Evaluator, evaluator: evaluator)
  SVMKit::Validation.check_params_boolean(return_train_score: return_train_score)
  @estimator = estimator
  @splitter = splitter
  @evaluator = evaluator
  @return_train_score = return_train_score
end

Instance Attribute Details

#estimator ⇒ Classifier (readonly)

Return the classifier of which performance is evaluated.

Returns:

  • (Classifier) 


26
27
28
 
# File 'lib/svmkit/model_selection/cross_validation.rb', line 26

def estimator
  @estimator
end

#evaluator ⇒ Evaluator (readonly)

Return the evaluator that calculates score.

Returns:

  • (Evaluator) 


34
35
36
 
# File 'lib/svmkit/model_selection/cross_validation.rb', line 34

def evaluator
  @evaluator
end

#return_train_score ⇒ Boolean (readonly)

Return the flag indicating whether to caculate the score of training dataset.

Returns:

  • (Boolean) 


38
39
40
 
# File 'lib/svmkit/model_selection/cross_validation.rb', line 38

def return_train_score
  @return_train_score
end

#splitter ⇒ Splitter (readonly)

Return the splitter that divides dataset.

Returns:

  • (Splitter) 


30
31
32
 
# File 'lib/svmkit/model_selection/cross_validation.rb', line 30

def splitter
  @splitter
end

Instance Method Details

#perform(x, y) ⇒ Hash

Perform the evalution of given classifier with cross-validation method.

Parameters:

  • x (Numo::DFloat) —

    (shape: [n_samples, n_features]) The dataset to be used to evaluate the estimator.

  • y (Numo::Int32 / Numo::DFloat) —

    (shape: [n_samples] / [n_samples, n_outputs]) The labels to be used to evaluate the classifier / The target values to be used to evaluate the regressor.

Returns:

  • (Hash) —

    The report summarizing the results of cross-validation.

    • :fit_time (Array<Float>) The calculation times of fitting the estimator for each split.

    • :test_score (Array<Float>) The scores of testing dataset for each split.

    • :train_score (Array<Float>) The scores of training dataset for each split. This option is nil if the return_train_score is false.



68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/svmkit/model_selection/cross_validation.rb', line 68

def perform(x, y)
  SVMKit::Validation.check_sample_array(x)
  if @estimator.is_a?(SVMKit::Base::Classifier)
    SVMKit::Validation.check_label_array(y)
    SVMKit::Validation.check_sample_label_size(x, y)
  end
  if @estimator.is_a?(SVMKit::Base::Regressor)
    SVMKit::Validation.check_tvalue_array(y)
    SVMKit::Validation.check_sample_tvalue_size(x, y)
  end
  # Initialize the report of cross validation.
  report = { test_score: [], train_score: nil, fit_time: [] }
  report[:train_score] = [] if @return_train_score
  # Evaluate the estimator on each split.
  @splitter.split(x, y).each do |train_ids, test_ids|
    # Split dataset into training and testing dataset.
    feature_ids = !kernel_machine? || train_ids
    train_x = x[train_ids, feature_ids]
    train_y = y.shape[1].nil? ? y[train_ids] : y[train_ids, true]
    test_x = x[test_ids, feature_ids]
    test_y = y.shape[1].nil? ? y[test_ids] : y[test_ids, true]
    # Fit the estimator.
    start_time = Time.now.to_i
    @estimator.fit(train_x, train_y)
    # Calculate scores and prepare the report.
    report[:fit_time].push(Time.now.to_i - start_time)
    if @evaluator.nil?
      report[:test_score].push(@estimator.score(test_x, test_y))
      report[:train_score].push(@estimator.score(train_x, train_y)) if @return_train_score
    elsif log_loss?
      report[:test_score].push(@evaluator.score(test_y, @estimator.predict_proba(test_x)))
      report[:train_score].push(@evaluator.score(train_y, @estimator.predict_proba(train_x))) if @return_train_score
    else
      report[:test_score].push(@evaluator.score(test_y, @estimator.predict(test_x)))
      report[:train_score].push(@evaluator.score(train_y, @estimator.predict(train_x))) if @return_train_score
    end
  end
  report
end