Class: Rouge::RegexLexer

Inherits:
Lexer
  • Object
show all
Defined in:
lib/rouge/lexer.rb

Defined Under Namespace

Classes: Rule, State, StateDSL

Constant Summary collapse

MAX_NULL_STEPS = 
5

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from Lexer

aliases, analyze_text, #debug, default_options, filenames, find, #get_tokens, guess, guess_by_filename, guess_by_mimetype, guess_by_source, #initialize, #lex, lex, mimetypes, #option, #options, register, tag

Constructor Details

This class inherits a constructor from Rouge::Lexer

Class Method Details

.[](name) ⇒ Object 



273
274
275
 
# File 'lib/rouge/lexer.rb', line 273

def self.[](name)
  get_state(name)
end

.get_state(name) ⇒ Object 



265
266
267
268
269
270
271
 
# File 'lib/rouge/lexer.rb', line 265

def self.get_state(name)
  return name if name.is_a? State

  state = states[name.to_s]
  raise "unknown state: #{name}" unless state
  state.load!
end

.start(&b) ⇒ Object 



256
257
258
 
# File 'lib/rouge/lexer.rb', line 256

def self.start(&b)
  start_procs << b
end

.start_procs ⇒ Object 



252
253
254
 
# File 'lib/rouge/lexer.rb', line 252

def self.start_procs
  @start_procs ||= []
end

.state(name, &b) ⇒ Object 



260
261
262
263
 
# File 'lib/rouge/lexer.rb', line 260

def self.state(name, &b)
  name = name.to_s
  states[name] = State.new(self, name, &b)
end

.states ⇒ Object 



248
249
250
 
# File 'lib/rouge/lexer.rb', line 248

def self.states
  @states ||= {}
end

Instance Method Details

#delegate(lexer, text = nil) ⇒ Object 



388
389
390
391
392
393
394
395
396
 
# File 'lib/rouge/lexer.rb', line 388

def delegate(lexer, text=nil)
  debug { "    delegating to #{lexer.inspect}" }
  text ||= @last_matches[0]

  lexer.lex(text, :continue => true) do |tok, val|
    debug { "    delegated token: #{tok.inspect}, #{val.inspect}" }
    token(tok, val)
  end
end

#get_state(name) ⇒ Object 



277
278
279
 
# File 'lib/rouge/lexer.rb', line 277

def get_state(name)
  self.class.get_state(name)
end

#group(tok) ⇒ Object 



384
385
386
 
# File 'lib/rouge/lexer.rb', line 384

def group(tok)
  token(tok, @last_matches[@group_count += 1])
end

#in_state?(state_name) ⇒ Boolean

Returns:

  • (Boolean) 


417
418
419
 
# File 'lib/rouge/lexer.rb', line 417

def in_state?(state_name)
  stack.map(&:name).include? state_name.to_s
end

#pop! ⇒ Object 



410
411
412
413
414
415
 
# File 'lib/rouge/lexer.rb', line 410

def pop!
  raise 'empty stack!' if stack.empty?

  debug { "    popping stack" }
  stack.pop
end

#push(state_name = nil, &b) ⇒ Object 



398
399
400
401
402
403
404
405
406
407
408
 
# File 'lib/rouge/lexer.rb', line 398

def push(state_name=nil, &b)
  # use the top of the stack by default
  if state_name || b
    push_state = state.relative_state(state_name, &b)
  else
    push_state = self.state
  end

  debug { "    pushing #{push_state.name}" }
  stack.push(push_state)
end

#reset! ⇒ Object 



289
290
291
292
293
294
295
 
# File 'lib/rouge/lexer.rb', line 289

def reset!
  @scan_state = nil

  self.class.start_procs.each do |pr|
    instance_eval(&pr)
  end
end

#run_callback(stream, &callback) ⇒ Object 



339
340
341
342
343
344
345
346
347
348
 
# File 'lib/rouge/lexer.rb', line 339

def run_callback(stream, &callback)
  Enumerator.new do |y|
    @output_stream = y
    @group_count = 0
    @last_matches = stream
    instance_exec(stream, &callback)
    @last_matches = nil
    @output_stream = nil
  end
end

#run_rule(rule, stream, &b) ⇒ Object 



319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
 
# File 'lib/rouge/lexer.rb', line 319

def run_rule(rule, stream, &b)
  case rule
  when String
    debug { "  entering mixin #{rule}" }
    res = step(get_state(rule), stream, &b)
    debug { "  exiting  mixin #{rule}" }
    res
  when Rule
    debug { "  trying #{rule.inspect}" }
    scan(stream, rule.re) do
      debug { "    got #{stream[0].inspect}" }

      run_callback(stream, &rule.callback).each do |tok, res|
        debug { "    yielding #{tok.to_s.inspect}, #{res.inspect}" }
        b.call(Token[tok], res)
      end
    end
  end
end

#scan(scanner, re, &b) ⇒ Object 



351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
 
# File 'lib/rouge/lexer.rb', line 351

def scan(scanner, re, &b)
  @null_steps ||= 0

  if @null_steps >= MAX_NULL_STEPS
    debug { "    too many scans without consuming the string!" }
    return false
  end

  scanner.scan(re)

  if scanner.matched?
    if scanner.matched_size == 0
      @null_steps += 1
    else
      @null_steps = 0
    end

    yield self
    return true
  end

  return false
end

#stack ⇒ Object 



281
282
283
 
# File 'lib/rouge/lexer.rb', line 281

def stack
  @stack ||= [get_state(:root)]
end

#state ⇒ Object 



285
286
287
 
# File 'lib/rouge/lexer.rb', line 285

def state
  stack.last or raise 'empty stack!'
end

#state?(state_name) ⇒ Boolean

Returns:

  • (Boolean) 


421
422
423
 
# File 'lib/rouge/lexer.rb', line 421

def state?(state_name)
  state_name.to_s == state.name
end

#step(state, stream, &b) ⇒ Object 



311
312
313
314
315
316
317
 
# File 'lib/rouge/lexer.rb', line 311

def step(state, stream, &b)
  state.rules.each do |rule|
    return true if run_rule(rule, stream, &b)
  end

  false
end

#stream_tokens(stream, &b) ⇒ Object 



297
298
299
300
301
302
303
304
305
306
307
308
309
 
# File 'lib/rouge/lexer.rb', line 297

def stream_tokens(stream, &b)
  until stream.eos?
    debug { "lexer: #{self.class.tag}" }
    debug { "stack: #{stack.map(&:name).inspect}" }
    debug { "stream: #{stream.peek(20).inspect}" }
    success = step(get_state(state), stream, &b)

    if !success
      debug { "    no match, yielding Error" }
      b.call(Token['Error'], stream.getch)
    end
  end
end

#token(tok, val = :__absent__) ⇒ Object 



375
376
377
378
379
380
381
382
 
# File 'lib/rouge/lexer.rb', line 375

def token(tok, val=:__absent__)
  val = @last_matches[0] if val == :__absent__
  val ||= ''

  raise 'no output stream' unless @output_stream

  @output_stream << [Token[tok], val]
end