Module: CTokenizer

Included in:
Cache, Lexer, LexerBase, Preprocessor::Parser, Preprocessor::Tokens
Defined in:
lib/dbc/ctokenizer.rb

Defined Under Namespace

Modules: Expression, Scoped Classes: CLexer, CPLexer, Cache, Error, Lexer, LexerBase, SkipMacros, Splitter

Constant Summary collapse

EOF_TOKEN = 
[false, false].freeze
C_RESERVED = 
c_reserved_symbol.dup
CP_RESERVED = 
cp_reserved_symbol.dup

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.check_token(t) ⇒ Object 



62
63
64
65
 
# File 'lib/dbc/ctokenizer.rb', line 62

def CTokenizer.check_token(t)
	raise "expecting a Array[2]: #{t.inspect}" \
		unless t.class <= Array and t.length == 2
end

.error(file, line, msg) ⇒ Object

Raises:



58
59
60
 
# File 'lib/dbc/ctokenizer.rb', line 58

def CTokenizer.error(file, line, msg)
	raise CTokenizer::Error.new(file, line), msg
end

.line_count(str) ⇒ Object 



67
68
69
70
71
 
# File 'lib/dbc/ctokenizer.rb', line 67

def CTokenizer.line_count(str)
	count = 0
	str.scan(Expression::NEWLINE) { count += 1 } if str.class == String
	count
end

.whitespace?(t) ⇒ Boolean

Returns:

  • (Boolean) 


73
74
75
76
77
78
79
80
 
# File 'lib/dbc/ctokenizer.rb', line 73

def CTokenizer.whitespace?(t)
	case t[0]
		when :SPACE, :NEWLINE, :COMMENT
			true
		else
			false
	end
end

Instance Method Details

#collect ⇒ Object 



113
114
115
116
117
118
119
 
# File 'lib/dbc/ctokenizer.rb', line 113

def collect
	ary = []
	until self.empty?
		ary << yield(self.shift)
	end
	ary
end

#each ⇒ Object 



106
107
108
109
110
111
 
# File 'lib/dbc/ctokenizer.rb', line 106

def each
	until self.empty?
		yield(self.shift)
	end
	self
end

#error(msg) ⇒ Object 



82
83
84
 
# File 'lib/dbc/ctokenizer.rb', line 82

def error(msg)
	CTokenizer.error(file, line, msg)
end

#parse_error(token) ⇒ Object 



90
91
92
 
# File 'lib/dbc/ctokenizer.rb', line 90

def parse_error(token)
	self.error("parse error on token: #{token}")
end

#to_a ⇒ Object 



98
99
100
101
102
103
104
 
# File 'lib/dbc/ctokenizer.rb', line 98

def to_a
	ary = []
	until self.empty?
		ary << self.shift
	end
	ary
end

#token_error(token) ⇒ Object 



86
87
88
 
# File 'lib/dbc/ctokenizer.rb', line 86

def token_error(token)
	self.error("unrecognized token: #{token}")
end

#warning(msg) ⇒ Object 



94
95
96
 
# File 'lib/dbc/ctokenizer.rb', line 94

def warning(msg)
	warn "#{file + ':' if file}#{line}: #{msg}"
end