Class: CodeLexer::LexedContent

Inherits:
Object
  • Object
show all
Defined in:
lib/code-lexer/lexer.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(tokens, abstractor = nil) ⇒ LexedContent

Returns a new instance of LexedContent.



40
41
42
43
44
45
 
# File 'lib/code-lexer/lexer.rb', line 40

def initialize(tokens, abstractor = nil)
    @tokens = tokens
    @abstractor = abstractor
    
    @abstractor.abstract!(@tokens) if @abstractor
end

Instance Attribute Details

#abstractor ⇒ Object (readonly)

Returns the value of attribute abstractor.



32
33
34
 
# File 'lib/code-lexer/lexer.rb', line 32

def abstractor
  @abstractor
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



31
32
33
 
# File 'lib/code-lexer/lexer.rb', line 31

def tokens
  @tokens
end

Class Method Details

.from_stream_string(stream, abstractor) ⇒ Object 



34
35
36
37
38
 
# File 'lib/code-lexer/lexer.rb', line 34

def self.from_stream_string(stream, abstractor)
    tokens = stream.split(" ").map { |t| Token.from_string(t) }
    abstractor.deabstract!(tokens)
    return LexedContent.new(tokens, abstractor)
end

.load(file_or_filename, lexdata_or_lexdata_filename = nil) ⇒ Object 



106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/code-lexer/lexer.rb', line 106

def self.load(file_or_filename, lexdata_or_lexdata_filename = nil)
    if file_or_filename.is_a?(String) && (lexdata_or_lexdata_filename.is_a?(String) || !lexdata_or_lexdata_filename)
        unless lexdata_or_lexdata_filename
            return self.load_filename(file_or_filename)
        else
            return self.load_filename(file_or_filename, lexdata_or_lexdata_filename)
        end
    elsif file_or_filename.is_a?(File) && lexdata_or_lexdata_filename.is_a?(File)
        return self.load_file(file_or_filename, lexdata_or_lexdata_filename)
    else
        raise "Unable to call with the provided input types: expected (String, String), (String), or (File, File)"
    end
end

.load_file(file, lexdata_file) ⇒ Object 



128
129
130
131
132
 
# File 'lib/code-lexer/lexer.rb', line 128

def self.load_file(file, lexdata_file)
    line = file.readline
    abstractor = Marshal.load(lexdata_file)
    return LexedContent.from_stream_string(line, abstractor)
end

.load_filename(filename, lexdata_filename = filename + ".lexdata") ⇒ Object 



120
121
122
123
124
125
126
 
# File 'lib/code-lexer/lexer.rb', line 120

def self.load_filename(filename, lexdata_filename = filename + ".lexdata")
    File.open(filename, "r") do |file|
        File.open(lexdata_filename, "rb") do |lexdata_file|
            return LexedContent.load_file(file, lexdata_file)
        end
    end
end

Instance Method Details

#dump(filename, mode = "w", force = false) ⇒ Object 



90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/code-lexer/lexer.rb', line 90

def dump(filename, mode = "w", force = false)
    if mode.downcase.include?("w") && !force
        if FileTest.exist?(filename) || FileTest.exist?(lexdata(filename))
            raise "Destination filename or lexdata filename already exist."
        end
    end
    
    File.open(filename, mode) do |f|
        f << self.token_stream + "\n"
    end
    
    File.open(lexdata(filename), "#{mode}b") do |f|
        f << Marshal.dump(@abstractor)
    end
end

#reconstruct ⇒ Object 



47
48
49
 
# File 'lib/code-lexer/lexer.rb', line 47

def reconstruct
    @tokens.map { |t| t.value.to_s }.join("")
end

#to_s ⇒ Object 



86
87
88
 
# File 'lib/code-lexer/lexer.rb', line 86

def to_s
    @tokens.map { |t| t.value }.join("")
end

#token_lines ⇒ Object 



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/code-lexer/lexer.rb', line 51

def token_lines
    result = []
    current_line = []
    @tokens.each do |t|
        if t.type == :newline
            result << current_line
            current_line = []
        else
            current_line << t
        end
    end
    
    result << current_line
    result.delete_if { |line| line.empty? }
    
    return result
end

#token_stream(abstractor = nil) ⇒ Object 



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/code-lexer/lexer.rb', line 69

def token_stream(abstractor = nil)
    result = []
    
    tokens = @tokens
    if abstractor
        tokens = tokens.map { |t| t.clone }
        tokens.each { |t| t.reset_abstraction }
        abstractor.abstract!(tokens)
    end
    
    tokens.each do |token|
        result << token.abstracted_value
    end
    
    return result.join(" ")
end