Class: Spellr::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/spellr/tokenizer.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(file, start_at: nil, skip_key: true) ⇒ Tokenizer

Returns a new instance of Tokenizer.



17
18
19
20
21
22
23
 
# File 'lib/spellr/tokenizer.rb', line 17

def initialize(file, start_at: nil, skip_key: true)
  @start_at = start_at || ColumnLocation.new(line_location: LineLocation.new(file))
  @file = file.is_a?(StringIO) || file.is_a?(IO) ? file : ::File.new(file)
  @file.pos = @start_at.line_location.byte_offset

  @line_tokenizer = LineTokenizer.new('', skip_key: skip_key)
end

Instance Attribute Details

#disabled ⇒ Object Also known as: disabled?

Returns the value of attribute disabled.



14
15
16
 
# File 'lib/spellr/tokenizer.rb', line 14

def disabled
  @disabled
end

#file ⇒ Object (readonly)

Returns the value of attribute file.



11
12
13
 
# File 'lib/spellr/tokenizer.rb', line 11

def file
  @file
end

#start_at ⇒ Object (readonly)

Returns the value of attribute start_at.



12
13
14
 
# File 'lib/spellr/tokenizer.rb', line 12

def start_at
  @start_at
end

Instance Method Details

#each_line_with_stats ⇒ Object

rubocop:disable Metrics/MethodLength



59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/spellr/tokenizer.rb', line 59

def each_line_with_stats # rubocop:disable Metrics/MethodLength
  char_offset = @start_at.line_location.char_offset
  byte_offset = @start_at.line_location.byte_offset

  file.each_line.with_index(@start_at.line_location.line_number) do |line, line_number|
    yield line, line_number, char_offset, byte_offset

    char_offset += line.length
    byte_offset += line.bytesize
  end
ensure
  file.close
end

#each_term(&block) ⇒ Object 



33
34
35
36
37
38
39
 
# File 'lib/spellr/tokenizer.rb', line 33

def each_term(&block)
  file.each_line do |line|
    prepare_tokenizer_for_line(line).each_term(&block)
  end
ensure
  file.close
end

#each_token(skip_term_proc: nil) ⇒ Object

rubocop:disable Metrics/MethodLength



41
42
43
44
45
46
47
48
49
 
# File 'lib/spellr/tokenizer.rb', line 41

def each_token(skip_term_proc: nil) # rubocop:disable Metrics/MethodLength
  each_line_with_stats do |line, line_number, char_offset, byte_offset|
    prepare_tokenizer_for_line(line).each_token(skip_term_proc: skip_term_proc) do |token|
      token.line = prepare_line(line, line_number, char_offset, byte_offset)

      yield token
    end
  end
end

#map(&block) ⇒ Object 



29
30
31
 
# File 'lib/spellr/tokenizer.rb', line 29

def map(&block)
  enum_for(:each_token).map(&block)
end

#normalized_terms ⇒ Object 



73
74
75
 
# File 'lib/spellr/tokenizer.rb', line 73

def normalized_terms
  enum_for(:each_term).map(&:spellr_normalize).uniq.sort
end

#prepare_line(line, line_number, char_offset, byte_offset) ⇒ Object 



51
52
53
54
55
56
57
 
# File 'lib/spellr/tokenizer.rb', line 51

def prepare_line(line, line_number, char_offset, byte_offset)
  line_location = LineLocation.new(
    file, line_number, char_offset: char_offset, byte_offset: byte_offset
  )
  column_location = ColumnLocation.new(line_location: line_location)
  Token.new(line, location: column_location)
end

#terms ⇒ Object 



25
26
27
 
# File 'lib/spellr/tokenizer.rb', line 25

def terms
  enum_for(:each_term).to_a
end