Class: Fmt::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/fmt/tokenizer.rb

Overview

Ruby source code token extractor

Uses Ripper from Ruby’s standard library

Examples:

Ripper token

[[lineno, column], type, token, state]
[[Integer, Integer], Symbol, String, Object]

See Also:

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(urtext) ⇒ Tokenizer

Constructor



19
20
21
22
 
# File 'lib/fmt/tokenizer.rb', line 19

def initialize(urtext)
  @urtext = urtext.to_s
  @tokens = []
end

Instance Attribute Details

#tokens ⇒ Object (readonly)

: Array – result of tokenization



25
26
27
 
# File 'lib/fmt/tokenizer.rb', line 25

def tokens
  @tokens
end

#urtext ⇒ Object (readonly)

: String – original source code



24
25
26
 
# File 'lib/fmt/tokenizer.rb', line 24

def urtext
  @urtext
end

Instance Method Details

#argument_tokens(start: 0) ⇒ Object

Returns the argument tokens



81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/fmt/tokenizer.rb', line 81

def argument_tokens(start: 0)
  starters = 0
  finishers = 0

  tokens[start..].each_with_object([]) do |token, memo|
    break memo if starters.positive? && finishers == starters

    starters += 1 if token.arguments_start?
    next if starters.zero?

    finishers += 1 if token.arguments_finish?
    memo << token
  end
end

#identifier_tokens(start: 0) ⇒ Object

Returns identifier tokens (typically method names)



39
40
41
42
43
44
 
# File 'lib/fmt/tokenizer.rb', line 39

def identifier_tokens(start: 0)
  tokens[start..].each_with_object([]) do |token, memo|
    break memo if token.arguments_start?
    memo << token if token.identifier?
  end
end

#key_tokens(start: 0) ⇒ Object

Returns key (named parameter) tokens



56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/fmt/tokenizer.rb', line 56

def key_tokens(start: 0)
  start = tokens[start..].find(&:key_start?)
  identifier = tokens[tokens.index(start)..].find(&:identifier?) if start
  finish = tokens[tokens.index(identifier)..].find(&:key_finish?) if identifier
  list = [start, identifier, finish].compact

  return [] unless list.size == 3
  return [] unless urtext.include?(list.map(&:value).join)

  list
end

#method_name_tokens(start: 0) ⇒ Object

Returns method tokens (identifiers and operators)



49
50
51
 
# File 'lib/fmt/tokenizer.rb', line 49

def method_name_tokens(start: 0)
  identifier_tokens(start: start) + operator_tokens(start: start)
end

#operator_tokens(start: 0) ⇒ Object

Returns operator tokens



71
72
73
74
75
76
 
# File 'lib/fmt/tokenizer.rb', line 71

def operator_tokens(start: 0)
  tokens[start..].each_with_object([]) do |token, memo|
    break memo if token.arguments_start?
    memo << token if token.operator?
  end
end

#tokenize ⇒ Object

Tokenizes the urtext (original source code)



29
30
31
32
33
34
 
# File 'lib/fmt/tokenizer.rb', line 29

def tokenize
  Ripper.lex(urtext).each do |token|
    tokens << Token.new(token)
  end
  tokens
end