Module: CTokenizer

Included in:
Cache, Lexer, LexerBase, Preprocessor::Parser, Preprocessor::Tokens
Defined in:
lib/dbc/ctokenizer.rb

Defined Under Namespace

Modules: Expression, Scoped Classes: CLexer, CPLexer, Cache, Error, Lexer, LexerBase, SkipMacros, Splitter

Constant Summary collapse

EOF_TOKEN = 
[false, false].freeze

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.check_token(t) ⇒ Object 



64
65
66
67
 
# File 'lib/dbc/ctokenizer.rb', line 64

def CTokenizer.check_token(t)
	raise "expecting a Array[2]: #{t.inspect}" \
		unless t.class <= Array and t.length == 2
end

.error(file, line, msg) ⇒ Object

Raises:



60
61
62
 
# File 'lib/dbc/ctokenizer.rb', line 60

def CTokenizer.error(file, line, msg)
	raise CTokenizer::Error.new(file, line), msg
end

.line_count(str) ⇒ Object 



69
70
71
72
73
 
# File 'lib/dbc/ctokenizer.rb', line 69

def CTokenizer.line_count(str)
	count = 0
	str.scan(Expression::NEWLINE) { count += 1 } if str.class == String
	count
end

.whitespace?(t) ⇒ Boolean

Returns:

  • (Boolean) 


75
76
77
78
79
80
81
82
 
# File 'lib/dbc/ctokenizer.rb', line 75

def CTokenizer.whitespace?(t)
	case t[0]
		when :SPACE, :NEWLINE, :COMMENT
			true
		else
			false
	end
end

Instance Method Details

#collect ⇒ Object 



115
116
117
118
119
120
121
 
# File 'lib/dbc/ctokenizer.rb', line 115

def collect
	ary = []
	until self.empty?
		ary << yield(self.shift)
	end
	ary
end

#each ⇒ Object 



108
109
110
111
112
113
 
# File 'lib/dbc/ctokenizer.rb', line 108

def each
	until self.empty?
		yield(self.shift)
	end
	self
end

#error(msg) ⇒ Object 



84
85
86
 
# File 'lib/dbc/ctokenizer.rb', line 84

def error(msg)
	CTokenizer.error(file, line, msg)
end

#parse_error(token) ⇒ Object 



92
93
94
 
# File 'lib/dbc/ctokenizer.rb', line 92

def parse_error(token)
	self.error("parse error on token: #{token}")
end

#to_a ⇒ Object 



100
101
102
103
104
105
106
 
# File 'lib/dbc/ctokenizer.rb', line 100

def to_a
	ary = []
	until self.empty?
		ary << self.shift
	end
	ary
end

#token_error(token) ⇒ Object 



88
89
90
 
# File 'lib/dbc/ctokenizer.rb', line 88

def token_error(token)
	self.error("unrecognized token: #{token}")
end

#warning(msg) ⇒ Object 



96
97
98
 
# File 'lib/dbc/ctokenizer.rb', line 96

def warning(msg)
	warn "#{file + ':' if file}#{line}: #{msg}"
end