Class: SQLTree::Tokenizer

Inherits:
Object
  • Object
show all
Includes:
Enumerable
Defined in:
lib/sql_tree/tokenizer.rb

Constant Summary collapse

OPERATOR_CHARS = 
/\=|<|>|!|\-|\+|\/|\*|\%/

Instance Method Summary collapse

Instance Method Details

#current_char ⇒ Object 



11
12
13
 
# File 'lib/sql_tree/tokenizer.rb', line 11

def current_char
  @current_char
end

#each_token(&block) ⇒ Object Also known as: each 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/sql_tree/tokenizer.rb', line 26

def each_token(&block)
  while next_char
    case current_char
    when /^\s?$/;        # whitespace, go to next token
    when '(';            yield(SQLTree::Token::LPAREN)
    when ')';            yield(SQLTree::Token::RPAREN)
    when '.';            yield(SQLTree::Token::DOT)
    when '.';            yield(SQLTree::Token::COMMA)        
    when /\d/;           tokenize_number(&block)
    when "'";            tokenize_quoted_string(&block)
    when OPERATOR_CHARS; tokenize_operator(&block)
    when /\w/;           tokenize_literal(&block)
    when '"';            tokenize_quoted_literal(&block)     # TODO: allow MySQL quoting mode   
    end      
  end
end

#next_char ⇒ Object 



19
20
21
22
 
# File 'lib/sql_tree/tokenizer.rb', line 19

def next_char
  @current_char_pos += 1
  @current_char = @string[@current_char_pos, 1]
end

#peek_char(amount = 1) ⇒ Object 



15
16
17
 
# File 'lib/sql_tree/tokenizer.rb', line 15

def peek_char(amount = 1)
  @string[@current_char_pos + amount, 1]
end

#tokenize(string) ⇒ Object 



5
6
7
8
9
 
# File 'lib/sql_tree/tokenizer.rb', line 5

def tokenize(string)
  @string = string
  @current_char_pos = -1    
  to_a
end

#tokenize_literal(&block) ⇒ Object 



45
46
47
48
49
50
51
52
53
54
 
# File 'lib/sql_tree/tokenizer.rb', line 45

def tokenize_literal(&block)
  literal = current_char
  literal << next_char while /[\w]/ =~ peek_char    
    
  if SQLTree::Token::KEYWORDS.include?(literal.downcase)
    yield(SQLTree::Token.const_get(literal.upcase))
  else
    yield(SQLTree::Token::Variable.new(literal))
  end
end

#tokenize_number(&block) ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/sql_tree/tokenizer.rb', line 56

def tokenize_number(&block)
  number = current_char
  dot_encountered = false
  while /\d/ =~ peek_char || (peek_char == '.' && !dot_encountered)
    dot_encountered = true if peek_char == '.'
    number << next_char
  end
  
  if dot_encountered
    yield(SQLTree::Token::Number.new(number.to_f))
  else
    yield(SQLTree::Token::Number.new(number.to_i))
  end
end

#tokenize_operator(&block) ⇒ Object 



88
89
90
91
92
93
94
95
96
 
# File 'lib/sql_tree/tokenizer.rb', line 88

def tokenize_operator(&block)
  operator = current_char
  if operator == '-' && /[\d\.]/ =~ peek_char 
    tokenize_number(&block)
  else
    operator << next_char if SQLTree::Token::OPERATORS.has_key?(operator + peek_char)
    yield(SQLTree::Token.const_get(SQLTree::Token::OPERATORS[operator].to_s.upcase))
  end
end

#tokenize_quoted_literal {|SQLTree::Token::Variable.new(literal)| ... } ⇒ Object

Yields:



79
80
81
82
83
84
85
 
# File 'lib/sql_tree/tokenizer.rb', line 79

def tokenize_quoted_literal(&block)
  literal = ''
  until next_char.nil? || current_char == '"' # TODO: allow MySQL quoting mode
    literal << (current_char == "\\" ? next_char : current_char)
  end
  yield(SQLTree::Token::Variable.new(literal))     
end

#tokenize_quoted_string {|SQLTree::Token::String.new(string)| ... } ⇒ Object

Yields:



71
72
73
74
75
76
77
 
# File 'lib/sql_tree/tokenizer.rb', line 71

def tokenize_quoted_string(&block)
  string = ''
  until next_char.nil? || current_char == "'"
    string << (current_char == "\\" ? next_char : current_char)
  end
  yield(SQLTree::Token::String.new(string))
end