Class: Winnow::Preprocessors::SourceCode

Inherits:
Winnow::Preprocessor show all
Defined in:
lib/winnow/preprocessors/source_code.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(language) ⇒ SourceCode

Returns a new instance of SourceCode.



8
9
10
 
# File 'lib/winnow/preprocessors/source_code.rb', line 8

def initialize(language)
  @lexer = Rouge::Lexer.find(language)
end

Instance Attribute Details

#lexer ⇒ Object (readonly)

Returns the value of attribute lexer.



6
7
8
 
# File 'lib/winnow/preprocessors/source_code.rb', line 6

def lexer
  @lexer
end

Instance Method Details

#preprocess(str) ⇒ Object 



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/winnow/preprocessors/source_code.rb', line 12

def preprocess(str)
  current_index = 0
  processed_chars = []

  lexer.lex(str).to_a.each do |token|
    type, chunk = token

    processed_chunk = case
    when type <= Rouge::Token::Tokens::Name
      'x'
    when type <= Rouge::Token::Tokens::Comment
      ''
    when type <= Rouge::Token::Tokens::Text
      ''
    else
      chunk
    end

    processed_chars += processed_chunk.chars.map do |c|
      [c, current_index]
    end

    current_index += chunk.length
  end

  processed_chars
end