Class: CodeLexer::Abstractor

Inherits:
Object
  • Object
show all
Defined in:
lib/code-lexer/abstractor.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(dictionary = []) ⇒ Abstractor

Returns a new instance of Abstractor.



7
8
9
 
# File 'lib/code-lexer/abstractor.rb', line 7

def initialize(dictionary=[])
    @dictionary = ["NOOP"] + dictionary
end

Instance Attribute Details

#dictionary ⇒ Object (readonly)

Returns the value of attribute dictionary.



5
6
7
 
# File 'lib/code-lexer/abstractor.rb', line 5

def dictionary
  @dictionary
end

Instance Method Details

#abstract!(tokens) ⇒ Object 



61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/code-lexer/abstractor.rb', line 61

def abstract!(tokens)
    if @abstract_identifiers
        identifier_tokens = tokens.select { |t| t.type == :identifier }
        identifiers = identifier_tokens.map { |id| id.value }.uniq
        
        identifiers.each do |id|
            if @dictionary.include?(id)
                abstracted_id = @dictionary.index(id)
            else
                abstracted_id = @dictionary.size
                @dictionary << id 
            end
                
            identifier_tokens.select { |t| t.value == id }.each do |matching_token|
                matching_token.abstracted_value = Token.special("ID#{abstracted_id}")
            end
        end
    end
    
    if @remove_comments
        tokens.delete_if { |t| t.type == :comment }
    elsif @abstract_comments
        tokens.select { |t| t.type == :comment }.each do |comment_token|
            comment_token.abstracted_value = Token.special("COMMENT")
        end
    end
    
    if @abstract_numbers
        tokens.select { |t| t.type == :number }.each do |number_token|
            number_token.abstracted_value = Token.special("NUMBER")
        end
    end
    
    if @abstract_strings
        tokens.select { |t| t.type == :string }.each do |string_token|
            string_token.abstracted_value = Token.special("STRING")
        end
    end
    
    if @remove_newlines
        tokens.delete_if { |t| t.type == :newline }
    end
    
    if @remove_spaces
        tokens.delete_if { |t| t.type == :space }
    elsif @abstract_spaces
        tokens.select { |t| t.type == :space }.each do |space_token|
            previous_index = tokens.index(space_token) - 1
            if previous_index < 0 || tokens[previous_index].type == :newline
                space_token.abstracted_value = Token.special("INDENTATION")
            else
                space_token.abstracted_value = Token.special("WHITESPACE")
            end
        end
    end
    
    return self
end

#abstract_comments ⇒ Object 



31
32
33
34
 
# File 'lib/code-lexer/abstractor.rb', line 31

def abstract_comments
    @abstract_comments = true
    return self
end

#abstract_everything ⇒ Object 



11
12
13
14
15
16
17
18
19
 
# File 'lib/code-lexer/abstractor.rb', line 11

def abstract_everything
    self.abstract_identifiers
    self.abstract_numbers
    self.abstract_comments
    self.abstract_strings
    self.abstract_spaces
    
    return self
end

#abstract_identifiers ⇒ Object 



21
22
23
24
 
# File 'lib/code-lexer/abstractor.rb', line 21

def abstract_identifiers
    @abstract_identifiers = true
    return self
end

#abstract_numbers ⇒ Object 



26
27
28
29
 
# File 'lib/code-lexer/abstractor.rb', line 26

def abstract_numbers
    @abstract_numbers = true
    return self
end

#abstract_spaces ⇒ Object 



41
42
43
44
 
# File 'lib/code-lexer/abstractor.rb', line 41

def abstract_spaces
    @abstract_spaces = true
    return self
end

#abstract_strings ⇒ Object 



36
37
38
39
 
# File 'lib/code-lexer/abstractor.rb', line 36

def abstract_strings
    @abstract_strings = true
    return self
end

#remove_comments ⇒ Object 



56
57
58
59
 
# File 'lib/code-lexer/abstractor.rb', line 56

def remove_comments
    @remove_comments = true
    return self
end

#remove_newlines ⇒ Object 



51
52
53
54
 
# File 'lib/code-lexer/abstractor.rb', line 51

def remove_newlines
    @remove_newlines = true
    return self
end

#remove_spaces ⇒ Object 



46
47
48
49
 
# File 'lib/code-lexer/abstractor.rb', line 46

def remove_spaces
    @remove_spaces = true
    return self
end