69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
# File 'lib/ext/string.rb', line 69
def tokenize
tokens = []
lit = /\~(?<literal>[^\.]+)/
regex = /\/(?<regex>.*?(?<!\\))\//
lookup = /\{\{(?<lookup>.*?)\}\}/
up = /(?<up>\^)/
path = /(?<path>[^\.\[]+)/
indexes = /(?<indexes>[\d\+\.,\s-]+)/
matcher = /#{lit}|#{regex}|#{lookup}|#{up}|#{path}(?:\[#{indexes}\])?/
matches(matcher) do |match|
if match.literal?
tokens << Mobj::Token.new(:literal, match.literal)
elsif match.lookup?
tokens << Mobj::Token.new(:lookup, match.lookup.tokenize)
elsif match.regex?
tokens << Mobj::Token.new(:regex, Regexp.new(match.regex))
elsif match.up?
tokens << Mobj::Token.new(:up)
elsif match.path?
eachs = match.path.split(/\s*,\s*/)
ors = match.path.split(/\s*\|\s*/)
ands = match.path.split(/\s*\&\s*/)
if eachs.size > 1
tokens << Mobj::Token.new(:each, eachs.map { |token| token.tokenize() })
elsif ands.size > 1
tokens << Mobj::Token.new(:all, ands.map { |token| token.tokenize() })
elsif ors.size > 1
tokens << Mobj::Token.new(:any, ors.map { |token| token.tokenize() })
end
unless ands.size + ors.size + eachs.size > 3
options = {}
index_matcher = /\s*(?<low>\d+)\s*(?:(?:\.\s*\.\s*(?<ex>\.)?\s*|-?)\s*(?<high>-?\d+|\+))?\s*/
options[:indexes] = match.indexes.matches(index_matcher).map do |index|
if index.high?
Range.new(index.low.to_i, (index.high == "+" ? -1 : index.high.to_i), index.ex?)
else
index.low.to_i
end
end if match.indexes?
if match.path[0] == '!'
tokens << Mobj::Token.new(:inverse, Token.new(:path, match.path[1..-1].sym, options))
else
tokens << Mobj::Token.new(:path, match.path.sym, options)
end
end
end
end
tokens.size == 1 ? tokens.first : Mobj::Token.new(:root, tokens)
end
|