Class: TokenizerRuby::Encoding

Inherits:
Object
  • Object
show all
Defined in:
lib/tokenizer_ruby/encoding.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(ids:, tokens:, offsets:, attention_mask:, type_ids: nil, special_tokens_mask: nil, word_ids: nil) ⇒ Encoding

Returns a new instance of Encoding.



8
9
10
11
12
13
14
15
16
 
# File 'lib/tokenizer_ruby/encoding.rb', line 8

def initialize(ids:, tokens:, offsets:, attention_mask:, type_ids: nil, special_tokens_mask: nil, word_ids: nil)
  @ids = ids
  @tokens = tokens
  @offsets = offsets
  @attention_mask = attention_mask
  @type_ids = type_ids
  @special_tokens_mask = special_tokens_mask
  @word_ids = word_ids
end

Instance Attribute Details

#attention_mask ⇒ Object (readonly)

Returns the value of attribute attention_mask.



5
6
7
 
# File 'lib/tokenizer_ruby/encoding.rb', line 5

def attention_mask
  @attention_mask
end

#ids ⇒ Object (readonly)

Returns the value of attribute ids.



5
6
7
 
# File 'lib/tokenizer_ruby/encoding.rb', line 5

def ids
  @ids
end

#offsets ⇒ Object (readonly)

Returns the value of attribute offsets.



5
6
7
 
# File 'lib/tokenizer_ruby/encoding.rb', line 5

def offsets
  @offsets
end

#special_tokens_mask ⇒ Object (readonly)

Returns the value of attribute special_tokens_mask.



5
6
7
 
# File 'lib/tokenizer_ruby/encoding.rb', line 5

def special_tokens_mask
  @special_tokens_mask
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



5
6
7
 
# File 'lib/tokenizer_ruby/encoding.rb', line 5

def tokens
  @tokens
end

#type_ids ⇒ Object (readonly)

Returns the value of attribute type_ids.



5
6
7
 
# File 'lib/tokenizer_ruby/encoding.rb', line 5

def type_ids
  @type_ids
end

#word_ids ⇒ Object (readonly)

Returns the value of attribute word_ids.



5
6
7
 
# File 'lib/tokenizer_ruby/encoding.rb', line 5

def word_ids
  @word_ids
end

Instance Method Details

#length ⇒ Object 



18
19
20
 
# File 'lib/tokenizer_ruby/encoding.rb', line 18

def length
  @ids.length
end