Class: Transformers::DebertaV2::DebertaV2ForSequenceClassification

Inherits:
DebertaV2PreTrainedModel show all
Defined in:
lib/transformers/models/deberta_v2/modeling_deberta_v2.rb

Instance Attribute Summary

Attributes inherited from PreTrainedModel

#config

Instance Method Summary collapse

Methods inherited from DebertaV2PreTrainedModel

#_init_weights

Methods inherited from PreTrainedModel

#_backward_compatibility_gradient_checkpointing, #_init_weights, #_initialize_weights, #base_model, #can_generate, #dequantize, #dummy_inputs, #framework, from_pretrained, #get_output_embeddings, #init_weights, #post_init, #prune_heads, #tie_weights, #warn_if_padding_and_no_attention_mask

Methods included from ClassAttribute

#class_attribute

Methods included from ModuleUtilsMixin

#device, #get_extended_attention_mask, #get_head_mask

Constructor Details

#initialize(config) ⇒ DebertaV2ForSequenceClassification

Returns a new instance of DebertaV2ForSequenceClassification.



930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
 
# File 'lib/transformers/models/deberta_v2/modeling_deberta_v2.rb', line 930

def initialize(config)
  super(config)

  num_labels = config.getattr("num_labels", 2)
  @num_labels = num_labels

  @deberta = DebertaV2Model.new(config)
  @pooler = ContextPooler.new(config)
  output_dim = @pooler.output_dim

  @classifier = Torch::NN::Linear.new(output_dim, num_labels)
  drop_out = config.getattr("cls_dropout", nil)
  drop_out = drop_out.nil? ? @config.hidden_dropout_prob : drop_out
  @dropout = StableDropout.new(drop_out)

  # Initialize weights and apply final processing
  post_init
end

Instance Method Details

#forward(input_ids: nil, attention_mask: nil, token_type_ids: nil, position_ids: nil, inputs_embeds: nil, labels: nil, output_attentions: nil, output_hidden_states: nil, return_dict: nil) ⇒ Object 



957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
 
# File 'lib/transformers/models/deberta_v2/modeling_deberta_v2.rb', line 957

def forward(
  input_ids: nil,
  attention_mask: nil,
  token_type_ids: nil,
  position_ids: nil,
  inputs_embeds: nil,
  labels: nil,
  output_attentions: nil,
  output_hidden_states: nil,
  return_dict: nil
)
  return_dict = !return_dict.nil? ? return_dict : @config.use_return_dict

  outputs = @deberta.(input_ids, token_type_ids: token_type_ids, attention_mask: attention_mask, position_ids: position_ids, inputs_embeds: inputs_embeds, output_attentions: output_attentions, output_hidden_states: output_hidden_states, return_dict: return_dict)

  encoder_layer = outputs[0]
  pooled_output = @pooler.(encoder_layer)
  pooled_output = @dropout.(pooled_output)
  logits = @classifier.(pooled_output)

  loss = nil
  if !labels.nil?
    if @config.problem_type.nil?
      if @num_labels == 1
        # regression task
        loss_fn = Torch::NN::MSELoss.new
        logits = logits.view(-1).to(labels.dtype)
        loss = loss_fn.(logits, labels.view(-1))
      elsif labels.dim == 1 || labels.size(-1) == 1
        label_index = (labels >= 0).nonzero
        labels = labels.long
        if label_index.size(0) > 0
          labeled_logits = Torch.gather(logits, 0, label_index.expand(label_index.size(0), logits.size(1)))
          labels = Torch.gather(labels, 0, label_index.view(-1))
          loss_fct = Torch::NN::CrossEntropyLoss.new
          loss = loss_fct.(labeled_logits.view(-1, @num_labels).float, labels.view(-1))
        else
          loss = Torch.tensor(0).to(logits)
        end
      else
        log_softmax = Torch::NN::LogSoftmax.new(-1)
        loss = -(log_softmax.(logits) * labels).sum(-1).mean
      end
    elsif @config.problem_type == "regression"
      loss_fct = Torch::NN::MSELoss.new
      if @num_labels == 1
        loss = loss_fct.(logits.squeeze, labels.squeeze)
      else
        loss = loss_fct.(logits, labels)
      end
    elsif @config.problem_type == "single_label_classification"
      loss_fct = Torch::NN::CrossEntropyLoss.new
      loss = loss_fct.(logits.view(-1, @num_labels), labels.view(-1))
    elsif @config.problem_type == "multi_label_classification"
      loss_fct = Torch::NN::BCEWithLogitsLoss.new
      loss = loss_fct.(logits, labels)
    end
  end
  if !return_dict
    output = [logits] + outputs[1..]
    return !loss.nil? ? [loss] + output : output
  end

  SequenceClassifierOutput.new(loss: loss, logits: logits, hidden_states: outputs.hidden_states, attentions: outputs.attentions)
end

#get_input_embeddings ⇒ Object 



949
950
951
 
# File 'lib/transformers/models/deberta_v2/modeling_deberta_v2.rb', line 949

def get_input_embeddings
  @deberta.get_input_embeddings
end

#set_input_embeddings(new_embeddings) ⇒ Object 



953
954
955
 
# File 'lib/transformers/models/deberta_v2/modeling_deberta_v2.rb', line 953

def set_input_embeddings(new_embeddings)
  @deberta.set_input_embeddings(new_embeddings)
end