Module: Puppet::Pops::Parser::LexerSupport

Included in:
HeredocSupport, Lexer2, SlurpSupport
Defined in:
lib/puppet/pops/parser/lexer_support.rb

Overview

This is an integral part of the Lexer. It is broken out into a separate module for maintainability of the code, and making the various parts of the lexer focused.

Defined Under Namespace

Classes: TokenValue

Constant Summary collapse

MM =
Puppet::Util::MultiMatch
MM_ANY =
MM::NOT_NIL
BOM_UTF_8 =
MM.new(0xEF, 0xBB, 0xBF,        MM_ANY)
BOM_UTF_16_1 =
MM.new(0xFE, 0xFF,              MM_ANY, MM_ANY)
BOM_UTF_16_2 =
MM.new(0xFF, 0xFE,              MM_ANY, MM_ANY)
BOM_UTF_32_1 =
MM.new(0x00, 0x00, 0xFE, 0xFF   )
BOM_UTF_32_2 =
MM.new(0xFF, 0xFE, 0x00, 0x00   )
BOM_UTF_1 =
MM.new(0xF7, 0x64, 0x4C,        MM_ANY)
BOM_UTF_EBCDIC =
MM.new(0xDD, 0x73, 0x66, 0x73   )
BOM_SCSU =
MM.new(0x0E, 0xFE, 0xFF,        MM_ANY)
BOM_BOCU =
MM.new(0xFB, 0xEE, 0x28,        MM_ANY)
BOM_GB_18030 =
MM.new(0x84, 0x31, 0x95, 0x33   )
LONGEST_BOM =
4

Instance Method Summary collapse

Instance Method Details

#assert_not_bom(content) ⇒ Object


167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
# File 'lib/puppet/pops/parser/lexer_support.rb', line 167

def assert_not_bom(content)
  name, size =
  case bom = get_bom(content)

  when BOM_UTF_32_1, BOM_UTF_32_2
    ['UTF-32', 4]

  when BOM_GB_18030
    ['GB-18030', 4]

  when BOM_UTF_EBCDIC
    ['UTF-EBCDIC', 4]

  when BOM_SCSU
    ['SCSU', 3]

  when BOM_UTF_8
    ['UTF-8', 3]

  when BOM_UTF_1
    ['UTF-1', 3]

  when BOM_BOCU
    ['BOCU', 3]

  when BOM_UTF_16_1, BOM_UTF_16_2
    ['UTF-16', 2]

  else
    return
  end

  lex_error_without_pos(
    Puppet::Pops::Issues::ILLEGAL_BOM,
    { :format_name   => name,
      :bytes  => "[#{bom.values[0,size].map {|b| "%X" % b}.join(" ")}]"
    })
end

#assert_numeric(value, pos) ⇒ Object

Asserts that the given string value is a float, or an integer in decimal, octal or hex form. An error is raised if the given value does not comply.


83
84
85
86
87
88
89
90
91
92
93
94
95
96
# File 'lib/puppet/pops/parser/lexer_support.rb', line 83

def assert_numeric(value, pos)
  if value =~ /^0[xX]/
    lex_error(Issues::INVALID_HEX_NUMBER, {:value => value}, pos)     unless value =~ /^0[xX][0-9A-Fa-f]+$/

  elsif value =~ /^0[^.]/
    lex_error(Issues::INVALID_OCTAL_NUMBER, {:value => value}, pos)   unless value =~ /^0[0-7]+$/

  elsif value =~ /^\d+[eE.]/
    lex_error(Issues::INVALID_DECIMAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+(?:\.\d+)?(?:[eE]-?\d+)?$/

  else
    lex_error(Issues::ILLEGAL_NUMBER, {:value => value}, pos) unless value =~ /^\d+$/
  end
end

#create_lex_error(issue, args = {}, pos = nil) ⇒ Puppet::ParseErrorWithIssue

Returns the created error


69
70
71
72
73
74
75
76
77
78
# File 'lib/puppet/pops/parser/lexer_support.rb', line 69

def create_lex_error(issue, args = {}, pos = nil)
  Puppet::ParseErrorWithIssue.new(
      issue.format(args),
      filename,
      line(pos),
      position(pos),
      nil,
      issue.issue_code,
      args)
end

#filenameObject


41
42
43
44
# File 'lib/puppet/pops/parser/lexer_support.rb', line 41

def filename
  file = @locator.file
  file.is_a?(String) && !file.empty? ? file : nil
end

#followed_byObject

Returns “<eof>” if at end of input, else the following 5 characters with n r t escaped


13
14
15
16
17
18
19
20
# File 'lib/puppet/pops/parser/lexer_support.rb', line 13

def followed_by
  return "<eof>" if @scanner.eos?
  result = @scanner.rest[0,5] + "..."
  result.gsub!("\t", '\t')
  result.gsub!("\n", '\n')
  result.gsub!("\r", '\r')
  result
end

#format_quote(q) ⇒ Object

Returns a quoted string using “ or ' depending on the given a strings's content


23
24
25
26
27
28
29
# File 'lib/puppet/pops/parser/lexer_support.rb', line 23

def format_quote(q)
  if q == "'"
    '"\'"'
  else
    "'#{q}'"
  end
end

#get_bom(content) ⇒ Object


206
207
208
209
210
211
212
213
214
215
216
217
# File 'lib/puppet/pops/parser/lexer_support.rb', line 206

def get_bom(content)
  # get 5 bytes as efficiently as possible (none of the string methods works since a bom consists of
  # illegal characters on most platforms, and there is no get_bytes(n). Explicit calls are faster than
  # looping with a lambda. The get_byte returns nil if there are too few characters, and they
  # are changed to spaces
  MM.new(
    (content.getbyte(0) || ' '),
    (content.getbyte(1) || ' '),
    (content.getbyte(2) || ' '),
    (content.getbyte(3) || ' ')
    )
end

#lex_error(issue, args = {}, pos = nil) ⇒ Object

Raises a Puppet::ParserErrorWithIssue with the given issue and arguments


37
38
39
# File 'lib/puppet/pops/parser/lexer_support.rb', line 37

def lex_error(issue, args = {}, pos=nil)
  raise create_lex_error(issue, args, pos)
end

#lex_error_without_pos(issue, args = {}) ⇒ Object

Raises a Puppet::LexError with the given message


32
33
34
# File 'lib/puppet/pops/parser/lexer_support.rb', line 32

def lex_error_without_pos(issue, args = {})
  raise Puppet::ParseErrorWithIssue.new(issue.format(args), nil, nil, nil, nil, issue.issue_code, args)
end

#lex_warning(issue, args = {}, pos = nil) ⇒ Object


54
55
56
57
58
59
60
61
62
63
# File 'lib/puppet/pops/parser/lexer_support.rb', line 54

def lex_warning(issue, args = {}, pos=nil)
  Puppet::Util::Log.create({
      :level => :warning,
      :message => issue.format(args),
      :issue_code => issue.issue_code,
      :file => filename,
      :line => line(pos),
      :pos => position(pos),
    })
end

#line(pos) ⇒ Object


46
47
48
# File 'lib/puppet/pops/parser/lexer_support.rb', line 46

def line(pos)
  @locator.line_for_offset(pos || @scanner.pos)
end

#position(pos) ⇒ Object


50
51
52
# File 'lib/puppet/pops/parser/lexer_support.rb', line 50

def position(pos)
  @locator.pos_on_line(pos || @scanner.pos)
end