Class: MailParser::Loose::Tokenizer

Inherits:
RFC2822::Scanner show all
Defined in:
lib/mailparser/loose.rb

Constant Summary

Constants inherited from RFC2822::Scanner

RFC2822::Scanner::ATEXT_RE, RFC2822::Scanner::CTEXT_RE, RFC2822::Scanner::DTEXT_RE, RFC2822::Scanner::QTEXT_RE, RFC2822::Scanner::TEXT_RE, RFC2822::Scanner::UTEXT_RE

Instance Attribute Summary

Attributes inherited from RFC2822::Scanner

#comments

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from RFC2822::Scanner

#cfws, #cfws_sub, #get_comment, #get_comment_by_id, #rest, #scan

Constructor Details

#initialize(str) ⇒ String

Returns str source string.



278
279
280
281
 
# File 'lib/mailparser/loose.rb', line 278

def initialize(str)
  @comments = []
  @ss = StringScanner.new(str)
end

Class Method Details

.token(str) ⇒ Object 



335
336
337
 
# File 'lib/mailparser/loose.rb', line 335

def self.token(str)
  Tokenizer.new(str).token
end

.token_received(str) ⇒ Object 



339
340
341
 
# File 'lib/mailparser/loose.rb', line 339

def self.token_received(str)
  Tokenizer.new(str).token_received
end

Instance Method Details

#token ⇒ Array<String>

tokenize

Returns:

  • (Array<String>) —

    tokens



285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
 
# File 'lib/mailparser/loose.rb', line 285

def token()
  token = []
  while @ss.rest? do
    if s = @ss.scan(/\s+/) then
      # ignore
    elsif s = @ss.scan(/\(/) then
      begin
        pos = @ss.pos
        cfws(@ss)
      rescue ParseError
        @ss.pos = pos
        token << s
      end
    elsif s = @ss.scan(/\"(\s*(\\[#{TEXT_RE}]|[#{QTEXT_RE}\x80-\xff]))*\s*\"/o) ||
        @ss.scan(/\[(\s*(\\[#{TEXT_RE}]|[#{DTEXT_RE}\x80-\xff]))*\s*\]/o) ||
        @ss.scan(/[#{ATEXT_RE}\x80-\xff]+/o)
      token << s
    else
      token << @ss.scan(/./)
    end
  end
  return token
end

#token_received ⇒ Array<String>

tokenize for Received field

Returns:

  • (Array<String>) —

    tokens



311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
 
# File 'lib/mailparser/loose.rb', line 311

def token_received()
  ret = []
  while @ss.rest? do
    if s = @ss.scan(/[\s]+/) then
      # ignore blank
    elsif s = @ss.scan(/\(/) then
      begin
        pos = @ss.pos
        cfws(@ss)
      rescue ParseError
        @ss.pos = pos
        ret.last << s unless ret.empty?
      end
    elsif s = @ss.scan(/\"([\s]*(\\[#{TEXT_RE}]|[#{QTEXT_RE}]))*[\s]*\"/o)
      ret << s
    elsif s = @ss.scan(/;/)
      ret << s
    else
      ret << @ss.scan(/[^\s\(\;]+/o)
    end
  end
  return ret
end