Class: UV::BufferedTokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/uv-rays/buffered_tokenizer.rb

Constant Summary collapse

DEFAULT_ENCODING = 
'ASCII-8BIT'.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options) ⇒ BufferedTokenizer

Returns a new instance of BufferedTokenizer.

Parameters:

  • options (Hash) 


23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/uv-rays/buffered_tokenizer.rb', line 23

def initialize(options)
    @delimiter  = options[:delimiter]
    @indicator  = options[:indicator]
    @msg_length = options[:msg_length]
    @size_limit = options[:size_limit]
    @min_length = options[:min_length] || 1
    @verbose    = options[:verbose] if @size_limit
    @encoding   = options[:encoding] || DEFAULT_ENCODING

    if @delimiter
        @delimiter.force_encoding(@encoding) if @delimiter.is_a?(String)
        @indicator.force_encoding(@encoding) if @indicator.is_a?(String)
        @extract_method = method(:delimiter_extract)
    elsif @indicator && @msg_length
        @indicator.force_encoding(@encoding) if @indicator.is_a?(String)
        @extract_method = method(:length_extract)
    else
        raise ArgumentError, 'no delimiter provided'
    end

    init_buffer
end

Instance Attribute Details

#delimiter ⇒ Object

Returns the value of attribute delimiter.



20
21
22
 
# File 'lib/uv-rays/buffered_tokenizer.rb', line 20

def delimiter
  @delimiter
end

#indicator ⇒ Object

Returns the value of attribute indicator.



20
21
22
 
# File 'lib/uv-rays/buffered_tokenizer.rb', line 20

def indicator
  @indicator
end

#size_limit ⇒ Object

Returns the value of attribute size_limit.



20
21
22
 
# File 'lib/uv-rays/buffered_tokenizer.rb', line 20

def size_limit
  @size_limit
end

#verbose ⇒ Object

Returns the value of attribute verbose.



20
21
22
 
# File 'lib/uv-rays/buffered_tokenizer.rb', line 20

def verbose
  @verbose
end

Instance Method Details

#empty? ⇒ Boolean

Returns:

  • (Boolean) 


73
74
75
 
# File 'lib/uv-rays/buffered_tokenizer.rb', line 73

def empty?
    @input.empty?
end

#extract(data) ⇒ Object

Extract takes an arbitrary string of input data and returns an array of tokenized entities, provided there were any available to extract.

Examples:


tokenizer.extract(data).
    map { |entity| Decode(entity) }.each { ... }

Parameters:

  • data (String) 


55
56
57
58
59
60
 
# File 'lib/uv-rays/buffered_tokenizer.rb', line 55

def extract(data)
    data.force_encoding(@encoding)
    @input << data

    @extract_method.call
end

#flush ⇒ String

Flush the contents of the input buffer, i.e. return the input buffer even though a token has not yet been encountered.

Returns:

  • (String) 


66
67
68
69
70
 
# File 'lib/uv-rays/buffered_tokenizer.rb', line 66

def flush
    buffer = @input
    reset
    buffer
end