Class: Dentaku::TokenScanner
- Inherits:
-
Object
- Object
- Dentaku::TokenScanner
- Extended by:
- StringCasing
- Defined in:
- lib/dentaku/token_scanner.rb
Class Attribute Summary collapse
-
.case_sensitive ⇒ Object
readonly
Returns the value of attribute case_sensitive.
Class Method Summary collapse
- .access ⇒ Object
- .array ⇒ Object
- .available_scanners ⇒ Object
- .boolean ⇒ Object
- .case_statement ⇒ Object
- .combinator ⇒ Object
- .comparator ⇒ Object
-
.datetime ⇒ Object
NOTE: Convert to DateTime as Array(Time) returns the parts of the time for some reason.
- .double_quoted_string ⇒ Object
- .function ⇒ Object
- .grouping ⇒ Object
- .hexadecimal ⇒ Object
- .identifier ⇒ Object
- .negate ⇒ Object
- .null ⇒ Object
- .numeric ⇒ Object
- .operator ⇒ Object
- .register_default_scanners ⇒ Object
- .register_scanner(id, scanner) ⇒ Object
- .register_scanners(scanner_ids) ⇒ Object
- .scanners(options = {}) ⇒ Object
- .scanners=(scanner_ids) ⇒ Object
- .single_quoted_string ⇒ Object
- .whitespace ⇒ Object
Instance Method Summary collapse
-
#initialize(category, regexp, converter = nil, condition = nil) ⇒ TokenScanner
constructor
A new instance of TokenScanner.
- #scan(string, last_token = nil) ⇒ Object
Methods included from StringCasing
Constructor Details
#initialize(category, regexp, converter = nil, condition = nil) ⇒ TokenScanner
Returns a new instance of TokenScanner.
10 11 12 13 14 15 |
# File 'lib/dentaku/token_scanner.rb', line 10 def initialize(category, regexp, converter = nil, condition = nil) @category = category @regexp = %r{\A(#{ regexp })}i @converter = converter @condition = condition || ->(*) { true } end |
Class Attribute Details
.case_sensitive ⇒ Object (readonly)
Returns the value of attribute case_sensitive.
31 32 33 |
# File 'lib/dentaku/token_scanner.rb', line 31 def case_sensitive @case_sensitive end |
Class Method Details
.access ⇒ Object
138 139 140 141 |
# File 'lib/dentaku/token_scanner.rb', line 138 def access names = { lbracket: '[', rbracket: ']' }.invert new(:access, '\[|\]', lambda { |raw| names[raw] }) end |
.array ⇒ Object
133 134 135 136 |
# File 'lib/dentaku/token_scanner.rb', line 133 def array names = { array_start: '{', array_end: '}', }.invert new(:array, '\{|\}|,', lambda { |raw| names[raw] }) end |
.available_scanners ⇒ Object
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 |
# File 'lib/dentaku/token_scanner.rb', line 33 def available_scanners [ :null, :whitespace, :datetime, # before numeric so it can pick up timestamps :numeric, :hexadecimal, :double_quoted_string, :single_quoted_string, :negate, :combinator, :operator, :grouping, :array, :access, :case_statement, :comparator, :boolean, :function, :identifier ] end |
.boolean ⇒ Object
162 163 164 |
# File 'lib/dentaku/token_scanner.rb', line 162 def boolean new(:logical, '(true|false)\b', lambda { |raw| raw.strip.downcase == 'true' }) end |
.case_statement ⇒ Object
143 144 145 146 |
# File 'lib/dentaku/token_scanner.rb', line 143 def case_statement names = { open: 'case', close: 'end', then: 'then', when: 'when', else: 'else' }.invert new(:case, '(case|end|then|when|else)\b', lambda { |raw| names[raw.downcase] }) end |
.combinator ⇒ Object
154 155 156 157 158 159 160 |
# File 'lib/dentaku/token_scanner.rb', line 154 def combinator names = { and: '&&', or: '||' }.invert new(:combinator, '(and|or|&&|\|\|)\s', lambda { |raw| norm = raw.strip.downcase names.fetch(norm) { norm.to_sym } }) end |
.comparator ⇒ Object
148 149 150 151 152 |
# File 'lib/dentaku/token_scanner.rb', line 148 def comparator names = { le: '<=', ge: '>=', ne: '!=', lt: '<', gt: '>', eq: '=' }.invert alternate = { ne: '<>', eq: '==' }.invert new(:comparator, '<=|>=|!=|<>|<|>|==|=', lambda { |raw| names[raw] || alternate[raw] }) end |
.datetime ⇒ Object
NOTE: Convert to DateTime as Array(Time) returns the parts of the time for some reason
88 89 90 |
# File 'lib/dentaku/token_scanner.rb', line 88 def datetime new(:datetime, /\d{2}\d{2}?-\d{1,2}-\d{1,2}( \d{1,2}:\d{1,2}:\d{1,2})? ?(Z|((\+|\-)\d{2}\:?\d{2}))?/, lambda { |raw| Time.parse(raw).to_datetime }) end |
.double_quoted_string ⇒ Object
102 103 104 |
# File 'lib/dentaku/token_scanner.rb', line 102 def double_quoted_string new(:string, '"[^"]*"', lambda { |raw| raw.gsub(/^"|"$/, '') }) end |
.function ⇒ Object
166 167 168 169 170 171 172 173 174 |
# File 'lib/dentaku/token_scanner.rb', line 166 def function new(:function, '\w+!?\s*\(', lambda do |raw| function_name = raw.gsub('(', '') [ Token.new(:function, function_name.strip.downcase.to_sym, function_name), Token.new(:grouping, :open, '(') ] end) end |
.grouping ⇒ Object
128 129 130 131 |
# File 'lib/dentaku/token_scanner.rb', line 128 def grouping names = { open: '(', close: ')', comma: ',' }.invert new(:grouping, '\(|\)|,', lambda { |raw| names[raw] }) end |
.hexadecimal ⇒ Object
98 99 100 |
# File 'lib/dentaku/token_scanner.rb', line 98 def hexadecimal new(:numeric, '(0x[0-9a-f]+)\b', lambda { |raw| raw[2..-1].to_i(16) }) end |
.identifier ⇒ Object
176 177 178 |
# File 'lib/dentaku/token_scanner.rb', line 176 def identifier new(:identifier, '[[[:word:]]\.]+\b', lambda { |raw| standardize_case(raw.strip) }) end |
.negate ⇒ Object
110 111 112 113 114 115 116 117 118 119 |
# File 'lib/dentaku/token_scanner.rb', line 110 def negate new(:operator, '-', lambda { |raw| :negate }, lambda { |last_token| last_token.nil? || last_token.is?(:operator) || last_token.is?(:comparator) || last_token.is?(:combinator) || last_token.value == :open || last_token.value == :comma }) end |
.null ⇒ Object
83 84 85 |
# File 'lib/dentaku/token_scanner.rb', line 83 def null new(:null, 'null\b') end |
.numeric ⇒ Object
92 93 94 95 96 |
# File 'lib/dentaku/token_scanner.rb', line 92 def numeric new(:numeric, '((?:\d+(\.\d+)?|\.\d+)(?:(e|E)(\+|-)?\d+)?)\b', lambda { |raw| raw =~ /\./ ? BigDecimal(raw) : raw.to_i }) end |
.operator ⇒ Object
121 122 123 124 125 126 |
# File 'lib/dentaku/token_scanner.rb', line 121 def operator names = { pow: '^', add: '+', subtract: '-', multiply: '*', divide: '/', mod: '%', bitor: '|', bitand: '&' }.invert new(:operator, '\^|\+|-|\*|\/|%|\||&', lambda { |raw| names[raw] }) end |
.register_default_scanners ⇒ Object
56 57 58 |
# File 'lib/dentaku/token_scanner.rb', line 56 def register_default_scanners register_scanners(available_scanners) end |
.register_scanner(id, scanner) ⇒ Object
66 67 68 |
# File 'lib/dentaku/token_scanner.rb', line 66 def register_scanner(id, scanner) @scanners[id] = scanner end |
.register_scanners(scanner_ids) ⇒ Object
60 61 62 63 64 |
# File 'lib/dentaku/token_scanner.rb', line 60 def register_scanners(scanner_ids) @scanners = scanner_ids.each_with_object({}) do |id, scanners| scanners[id] = self.send(id) end end |
.scanners(options = {}) ⇒ Object
74 75 76 77 |
# File 'lib/dentaku/token_scanner.rb', line 74 def scanners( = {}) @case_sensitive = .fetch(:case_sensitive, false) @scanners.values end |
.scanners=(scanner_ids) ⇒ Object
70 71 72 |
# File 'lib/dentaku/token_scanner.rb', line 70 def scanners=(scanner_ids) @scanners.select! { |k, v| scanner_ids.include?(k) } end |
.single_quoted_string ⇒ Object
106 107 108 |
# File 'lib/dentaku/token_scanner.rb', line 106 def single_quoted_string new(:string, "'[^']*'", lambda { |raw| raw.gsub(/^'|'$/, '') }) end |
.whitespace ⇒ Object
79 80 81 |
# File 'lib/dentaku/token_scanner.rb', line 79 def whitespace new(:whitespace, '\s+') end |
Instance Method Details
#scan(string, last_token = nil) ⇒ Object
17 18 19 20 21 22 23 24 25 26 27 28 |
# File 'lib/dentaku/token_scanner.rb', line 17 def scan(string, last_token = nil) if (m = @regexp.match(string)) && @condition.call(last_token) value = raw = m.to_s value = @converter.call(raw) if @converter return Array(value).map do |v| Token === v ? v : Token.new(@category, v, raw) end end false end |