Class: Markleft::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/markleft.rb

Overview

—————– Tokenizer Module —————– #

Constant Summary collapse

TOKEN_TYPES = 
[
  [:heading1, /^# .+/],
  [:heading2, /^## .+/],
  [:heading3, /^### .+/],
  [:heading4, /^#### .+/],
  [:heading5, /^##### .+/],
  [:heading6, /^###### .+/],
  [:strikethrough, /((?<!\s)~~(?!\s)(?:[^~]+?|(?:[^~]*(?:(?:~[^~]*){2})+?)+?)(?<!\s)~~)/],
  [:bold, /((?<!\s)\*\*(?!\s)(?:[^*]+?|(?:[^*]*(?:(?:\*[^*]*){2})+?)+?)(?<!\s)\*\*)/],
  [:italics, /((?<!\s)\*(?!\s)(?:(?:[^**]*(?:(?:\*\*[^**]*){2})+?)+?|[^**]+?)(?<!\s)\*)/],
  [:newline, /(\r\n|\r|\n)/],
  [:text, /[^*\n\r]+/]
].freeze

Instance Method Summary collapse

Constructor Details

#initialize(markleft) ⇒ Tokenizer

Returns a new instance of Tokenizer.



25
26
27
 
# File 'lib/markleft.rb', line 25

def initialize(markleft)
  @code = markleft
end

Instance Method Details

#tokenize ⇒ Object 



29
30
31
32
33
34
35
36
 
# File 'lib/markleft.rb', line 29

def tokenize
  tokens = []
  until @code.empty?
    tokens << tokenize_one_token
    @code = @code.strip
  end
  tokens
end

#tokenize_one_token ⇒ Object 



38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/markleft.rb', line 38

def tokenize_one_token
  TOKEN_TYPES.each do |type, re|
    re = /\A(#{re})/
    next unless @code =~ re

    value = ::Regexp.last_match(1)
    @code = @code[value.length..]
    return Token.new(type, value)
  end
  raise "Unexpected character #{@code.inspect}"
end