Class: SQLTree::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/active_record/turntable/sql_tree_patch.rb

Instance Method Summary collapse

Instance Method Details

#each_token(&block) ⇒ Object Also known as: each

Note:

Override to handle x’..‘ binary string



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/active_record/turntable/sql_tree_patch.rb', line 40

def each_token(&block) # :yields: SQLTree::Token

  while next_char
    case current_char
    when /^\s?$/;        # whitespace, go to next character
    when '(';            handle_token(SQLTree::Token::LPAREN, &block)
    when ')';            handle_token(SQLTree::Token::RPAREN, &block)
    when '.';            handle_token(SQLTree::Token::DOT, &block)
    when ',';            handle_token(SQLTree::Token::COMMA, &block)
    when /\d/;           tokenize_number(&block)
    when "'";            tokenize_quoted_string(&block)
    when 'E', 'x', 'X';  tokenize_possible_escaped_string(&block)
    when /\w/;           tokenize_keyword(&block)
    when OPERATOR_CHARS; tokenize_operator(&block)
    when SQLTree.identifier_quote_char; tokenize_quoted_identifier(&block)
    end
  end

  # Make sure to yield any tokens that are still stashed on the queue.
  empty_keyword_queue!(&block)
end

#tokenize_possible_escaped_string(&block) ⇒ Object 



63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/active_record/turntable/sql_tree_patch.rb', line 63

def tokenize_possible_escaped_string(&block)
  if peek_char == "'"
    token = case current_char
            when 'E'
              SQLTree::Token::STRING_ESCAPE
            when 'x', 'X'
              SQLTree::Token::BINARY_ESCAPE
            end
    handle_token(token, &block)
  else
    tokenize_keyword(&block)
  end
end

#tokenize_quoted_string(&block) ⇒ Object

:yields: SQLTree::Token::String



31
32
33
34
35
36
37
 
# File 'lib/active_record/turntable/sql_tree_patch.rb', line 31

def tokenize_quoted_string(&block) # :yields: SQLTree::Token::String
  string = ''
  until next_char.nil? || current_char == "'"
    string << (current_char == "\\" ? instance_eval("%@\\#{next_char.gsub('@', '\@')}@") : current_char)
  end
  handle_token(SQLTree::Token::String.new(string), &block)
end