7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
|
# File 'lib/rubypitaya/app-template/vendor/bundle/ruby/3.1.0/gems/cucumber-cucumber-expressions-15.2.0/lib/cucumber/cucumber_expressions/cucumber_expression_tokenizer.rb', line 7
def tokenize(expression)
@expression = expression
tokens = []
@buffer = []
previous_token_type = TokenType::START_OF_LINE
treat_as_text = false
@escaped = 0
@buffer_start_index = 0
codepoints = expression.codepoints
if codepoints.empty?
tokens.push(Token.new(TokenType::START_OF_LINE, '', 0, 0))
end
codepoints.each do |codepoint|
if !treat_as_text && Token.is_escape_character(codepoint)
@escaped += 1
treat_as_text = true
next
end
current_token_type = token_type_of(codepoint, treat_as_text)
treat_as_text = false
if should_create_new_token?(previous_token_type, current_token_type)
token = convert_buffer_to_token(previous_token_type)
previous_token_type = current_token_type
@buffer.push(codepoint)
tokens.push(token)
else
previous_token_type = current_token_type
@buffer.push(codepoint)
end
end
if @buffer.length > 0
token = convert_buffer_to_token(previous_token_type)
tokens.push(token)
end
raise TheEndOfLineCannotBeEscaped.new(expression) if treat_as_text
tokens.push(Token.new(TokenType::END_OF_LINE, '', codepoints.length, codepoints.length))
tokens
end
|