Class: SQLPP::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/sqlpp/tokenizer.rb

Defined Under Namespace

Classes: EOFError, Exception, Token, UnexpectedCharacter

Constant Summary collapse

KEYWORDS = 
%w(
  and
  as
  asc
  between
  by
  case
  cross
  desc
  else
  end
  first
  from
  full
  group
  having
  ilike
  in
  inner
  is
  join
  last
  left
  like
  not
  null
  nulls
  on
  or
  order
  outer
  right
  select
  then
  when
  where
)
KEYWORDS_REGEX = 
Regexp.new('\b(' + KEYWORDS.join('|') + ')\b', Regexp::IGNORECASE)

Instance Method Summary collapse

Constructor Details

#initialize(string) ⇒ Tokenizer

Returns a new instance of Tokenizer.



52
53
54
55
 
# File 'lib/sqlpp/tokenizer.rb', line 52

def initialize(string)
  @scanner = StringScanner.new(string)
  @buffer = []
end

Instance Method Details

#_scan ⇒ Object 



74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/sqlpp/tokenizer.rb', line 74

def _scan
  pos = @scanner.pos

  if @scanner.eos?
    Token.new(:eof, nil, pos)
  elsif (key = @scanner.scan(KEYWORDS_REGEX))
    Token.new(:key, key.downcase.to_sym, pos)
  elsif (num = @scanner.scan(/\d+(?:\.\d+)?/))
    Token.new(:lit, num, pos)
  elsif (id = @scanner.scan(/\w+/))
    Token.new(:id, id, pos)
  elsif (punct = @scanner.scan(/<=|<>|!=|>=/))
    Token.new(:punct, punct, pos)
  elsif (punct = @scanner.scan(/[<>=\(\).*,\/+\-]/))
    Token.new(:punct, punct, pos)
  elsif @scanner.scan(/"/)
    contents = _scan_to_delim('"', pos)
    Token.new(:id, "\"#{contents}\"", pos)
  elsif @scanner.scan(/'/)
    contents = _scan_to_delim("'", pos)
    Token.new(:lit, "'#{contents}'", pos)
  elsif (space = @scanner.scan(/\s+/))
    Token.new(:space, space, pos)
  else
    raise UnexpectedCharacter, @scanner.rest
  end
end

#_scan_to_delim(delim, pos) ⇒ Object 



102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/sqlpp/tokenizer.rb', line 102

def _scan_to_delim(delim, pos)
  string = ""
  loop do
    ch = @scanner.getch

    if delim == '"' && ch == "\\"
      ch << @scanner.getch
    elsif delim == "'" && ch == "'"
      ch << @scanner.getch if @scanner.peek(1) == "'"
    end

    case ch
    when nil then
      raise EOFError, "end of input reached in string started at #{pos} with #{delim.inspect}"
    when delim then
      return string
    else
      string << ch
    end
  end
end

#next ⇒ Object 



57
58
59
60
61
62
63
 
# File 'lib/sqlpp/tokenizer.rb', line 57

def next
  if @buffer.any?
    @buffer.pop
  else
    _scan
  end
end

#peek ⇒ Object 



65
66
67
 
# File 'lib/sqlpp/tokenizer.rb', line 65

def peek
  push(self.next)
end

#push(token) ⇒ Object 



69
70
71
72
 
# File 'lib/sqlpp/tokenizer.rb', line 69

def push(token)
  @buffer.push(token)
  token
end