Class: Canopus::Language::BackgroundAnalysis

Inherits:
Object
  • Object
show all
Defined in:
lib/canopus/language/background_analysis.rb

Defined Under Namespace

Classes: Job, Scheduler, Snapshot

Constant Summary collapse

EMPTY = 
[].freeze
SOURCE_LIMIT = 
2 << 20
LINE_LIMIT = 
16 << 10
ROW_LIMIT = 
256
CACHE_ROWS = 
512
SELECTION_LIMIT = 
256
SELECTION_RANGE_LIMIT = 
10_000

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(document) ⇒ BackgroundAnalysis

Returns a new instance of BackgroundAnalysis.



16
17
18
19
20
21
22
 
# File 'lib/canopus/language/background_analysis.rb', line 16

def initialize(document)
  @document, @buffer = document, document.buffer
  @id = "#{Process.pid}:#{object_id}"
  @tokens, @stale, @provisional, @generation = {}, {}, {}, 0
  @syntax_rows = {}
  @disabled = @buffer.rope.respond_to?(:lazy?) && @buffer.rope.lazy?
end

Instance Attribute Details

#analysis_error ⇒ Object (readonly)

Returns the value of attribute analysis_error.



14
15
16
 
# File 'lib/canopus/language/background_analysis.rb', line 14

def analysis_error
  @analysis_error
end

Class Method Details

.boundary(rope, offset) ⇒ Object 



214
215
216
217
218
219
220
 
# File 'lib/canopus/language/background_analysis.rb', line 214

def self.boundary(rope, offset)
  rope.point_at(offset)
  offset
rescue RangeError
  offset -= 1
  retry
end

.decode(response, prior_syntax: nil) ⇒ Object 



259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
 
# File 'lib/canopus/language/background_analysis.rb', line 259

def self.decode(response, prior_syntax: nil)
  tokens = response.fetch("tokens").to_h do |row, pairs|
    [row, pairs.map { |name, value| [name.freeze, value.freeze].freeze }.freeze]
  end.freeze
  raw = response["syntax"]
  syntax = if raw
    outline = raw.fetch("outline").map do |name, kind, first, last, start, finish, depth|
      Language::Symbol.new(name.freeze, kind.to_sym, (first...last).freeze, (start...finish).freeze, depth)
    end.freeze
    bracket_pairs = raw.fetch("brackets").map do |open_first, open_last, open_row, close_first, close_last, close_row, depth|
      Language::Bracket.new((open_first...open_last).freeze, (close_first...close_last).freeze,
        open_row, close_row, depth).freeze
    end.freeze
    brackets = bracket_pairs.each_with_object({}) do |pair, result|
      range = (pair.open_range.begin...pair.close_range.end).freeze
      result[pair.open_range.begin] = result[pair.close_range.begin] = range
    end
    {outline: outline, diagnostics: raw.fetch("diagnostics").map { |first, last, message| {range: (first...last).freeze, message: message.freeze, severity: 1}.freeze }.freeze,
      folds: raw.fetch("folds").map { |first, last| (first...last).freeze }.freeze,
      brackets: brackets.freeze, bracket_pairs: bracket_pairs,
      structure_regions: raw.fetch("structure_regions").map { |first, last, kind| {start_line: first, end_line: last, kind: kind.to_sym}.freeze }.freeze,
      complete: response.fetch("complete")}.freeze
  end
  syntax = merge_syntax(prior_syntax, syntax) if prior_syntax && syntax && !syntax[:complete]
  values = response.fetch("selections", [])
  unless values.is_a?(Array) && values.length <= SELECTION_LIMIT
    raise Error, "invalid background selection ranges"
  end
  seen, total = {}, 0
  selections = values.to_h do |index, ranges|
    total += ranges.length if ranges.is_a?(Array)
    unless index.is_a?(Integer) && index.between?(0, SELECTION_LIMIT - 1) && !seen.key?(index) &&
        ranges.is_a?(Array) && total <= SELECTION_RANGE_LIMIT && ranges.all? do |range|
          range.is_a?(Array) && range.length == 2 && range.all? { |value| value.is_a?(Integer) && value >= 0 } &&
            range[0] <= range[1]
        end
      raise Error, "invalid background selection ranges"
    end
    seen[index] = true
    [index, ranges.map { |first, last| (first...last).freeze }.freeze]
  end.freeze
  {tokens: tokens, syntax: syntax, selections: selections}.freeze
end

.line(rope, row) ⇒ Object 



207
208
209
210
211
212
 
# File 'lib/canopus/language/background_analysis.rb', line 207

def self.line(rope, row)
  first = rope.line_start(row)
  last = row + 1 < rope.line_count ? rope.line_start(row + 1) : rope.bytesize
  ending = boundary(rope, [last, first + LINE_LIMIT].min)
  rope.byteslice(first, ending - first).to_s.freeze
end

.merge_syntax(previous, current) ⇒ Object

Decoding and bounded merging happen on a preparation thread, never in paint/poll. Overlapping context windows must not duplicate symbols.



305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
 
# File 'lib/canopus/language/background_analysis.rb', line 305

def self.merge_syntax(previous, current)
  return previous if previous[:complete]
  outlines = (previous[:outline] + current[:outline]).uniq { |item| [item.name, item.kind, item.range, item.selection] }
  bracket_pairs = (previous.fetch(:bracket_pairs, []) + current.fetch(:bracket_pairs, []))
    .uniq { |pair| [pair.open_range, pair.close_range] }.first(8192)
  brackets = bracket_pairs.each_with_object({}) do |pair, result|
    range = (pair.open_range.begin...pair.close_range.end).freeze
    result[pair.open_range.begin] = result[pair.close_range.begin] = range
  end
  structure_regions = (previous.fetch(:structure_regions, []) + current.fetch(:structure_regions, []))
    .uniq { |region| [region[:start_line], region[:end_line], region[:kind]] }.first(10_000)
  {outline: outlines.sort_by { |item| item.selection.begin }.first(10_000).freeze,
    diagnostics: (previous[:diagnostics] + current[:diagnostics]).uniq.first(1000).freeze,
    folds: (previous[:folds] + current[:folds]).uniq.sort_by(&:begin).first(10_000).freeze,
    brackets: brackets.freeze, bracket_pairs: bracket_pairs.freeze,
    structure_regions: structure_regions.freeze,
    complete: false}.freeze
end

.prepare(snapshot, job) ⇒ Object 



222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
 
# File 'lib/canopus/language/background_analysis.rb', line 222

def self.prepare(snapshot, job)
  rope, first, last = snapshot.rope, snapshot.first, snapshot.last
  complete = rope.bytesize <= 1 << 20
  from, to, context = 0, rope.bytesize, 0
  unless complete
    context = rope.bytesize <= 10 << 20 ? 500 : 0
    start_row = [first - context, 0].max
    finish_row = [last + context + 1, rope.line_count].min
    requested_end = last + 1 < rope.line_count ? rope.line_start(last + 1) : rope.bytesize
    while start_row < first && requested_end - rope.line_start(start_row) > SOURCE_LIMIT
      job.check!
      start_row += 1
    end
    from = rope.line_start(start_row)
    to = finish_row < rope.line_count ? rope.line_start(finish_row) : rope.bytesize
    to = boundary(rope, [to, from + SOURCE_LIMIT].min)
  end
  # A capped snapshot may end before later requested rows, or exactly at
  # the start of a row whose content was not copied. Never cache that row
  # as an empty, supposedly current token result; the next batch reads it.
  end_row = rope.point_at(to).row
  end_row -= 1 if to < rope.bytesize && rope.line_start(end_row) == to
  last = [last, end_row].min
  job.check!
  source = rope.byteslice(from, to - from).to_s.freeze
  selection_request = snapshot.syntax.is_a?(Hash) ? snapshot.syntax : {}
  syntax = selection_request.fetch(:syntax, snapshot.syntax == true)
  selections = Array(selection_request[:selections]).filter_map do |index, row, column, _offset|
    [index, row, column] if row.between?(rope.point_at(from).row, end_row)
  end
  {"id" => snapshot.id, "version" => snapshot.version, "name" => snapshot.name,
    "lexer" => snapshot.lexer, "source" => source, "base" => from,
    "base_line" => rope.point_at(from).row, "first" => first, "last" => last,
    "syntax" => syntax, "selections" => selections.freeze,
    "complete" => complete, "context" => context}.freeze
end

Instance Method Details

#bracket_at(offset) ⇒ Object 



87
 
# File 'lib/canopus/language/background_analysis.rb', line 87

def bracket_at(offset) = (request(syntax: true); @syntax && @syntax[:brackets][offset])

#brackets(rows = nil) ⇒ Object 



88
89
90
91
92
 
# File 'lib/canopus/language/background_analysis.rb', line 88

def brackets(rows = nil)
  request(syntax: true)
  values = @syntax ? @syntax[:bracket_pairs] : EMPTY
  rows ? values.select { |pair| pair.close_row >= rows.begin && pair.open_row < rows.end } : values
end

#diagnostics ⇒ Object 



85
 
# File 'lib/canopus/language/background_analysis.rb', line 85

def diagnostics = (request(syntax: true); @syntax ? @syntax[:diagnostics] : EMPTY)

#dispose ⇒ Object 



191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/canopus/language/background_analysis.rb', line 191

def dispose
  return if @disposed
  @disposed = true
  @job&.cancel
  @job = nil
  Scheduler.release(@scheduler) if @scheduler
  @scheduler = nil
  @tokens.clear
  @stale.clear
  @provisional.clear
  @syntax_rows.clear
  @syntax = nil
  @selection_positions = @selection_results = nil
  @analysis_error = nil
end

#fold_ranges ⇒ Object 



86
 
# File 'lib/canopus/language/background_analysis.rb', line 86

def fold_ranges = (request(syntax: true); @syntax ? @syntax[:folds] : EMPTY)

#invalidate(patch) ⇒ Object 



116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/canopus/language/background_analysis.rb', line 116

def invalidate(patch)
  @job&.cancel
  @job = @syntax = nil
  @selection_positions = @selection_results = nil
  @syntax_rows.clear
  if patch.is_a?(Patch::Reload)
    @tokens.clear
    @stale.clear
  else
    (patch.is_a?(Patch::Composite) ? patch.patches : [patch]).each { |item| retain_unchanged_rows(item) }
  end
  @provisional.clear
  @generation += 1
  @failed = false
  @analysis_error = nil
  @disabled = @buffer.rope.respond_to?(:lazy?) && @buffer.rope.lazy?
  @rows = @rows&.map { |row| [row, @buffer.line_count - 1].min }&.uniq&.freeze
  @range = [@rows.first, @rows.last].freeze if @rows
  @dirty = !!@rows && !@disabled
end

#outline ⇒ Object 



84
 
# File 'lib/canopus/language/background_analysis.rb', line 84

def outline = (request(syntax: true); @syntax ? @syntax[:outline] : EMPTY)

#pending? ⇒ Boolean

Returns:

  • (Boolean) 


80
 
# File 'lib/canopus/language/background_analysis.rb', line 80

def pending? = !@disposed && !@disabled && (!!@job || !!@dirty)

#poll ⇒ Object 



137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
 
# File 'lib/canopus/language/background_analysis.rb', line 137

def poll
  return false unless pending?
  changed = false
  if @job&.future&.done?
    job, @job = @job, nil
    result = job.future.await(timeout: 0)
    if @submitted_version == @buffer.version && @submitted_generation == @generation
      result[:tokens].each { |row, tokens| @tokens[row] = tokens; @stale.delete(row) }
      if @selection_results
        result[:selections].each { |index, ranges| @selection_results[index] = ranges }
        if @selection_results.values.sum(&:length) > SELECTION_RANGE_LIMIT
          raise Error, "too many background selection ranges"
        end
      end
      while @tokens.length > CACHE_ROWS
        @tokens.delete(@tokens.each_key.find { |row| !@rows.include?(row) })
      end
      @stale.shift while @stale.length + @tokens.length > CACHE_ROWS
      if result[:syntax]
        @syntax = result[:syntax]
        result[:tokens].each_key { |row| @syntax_rows[row] = true }
      end
      @dirty = missing_work?
      changed = true
    end
  end
  if @dirty && !@job
    @scheduler ||= Scheduler.acquire
    # A folded viewport can contain distant source rows. Fill one missing
    # contiguous run at a time without paint changing/cancelling its job.
    rows = work_rows
    missing = rows.select { |row| row_needed?(row) }
    missing = rows if missing.empty?
    first = last = missing.first
    missing.drop(1).each do |row|
      break unless row == last + 1
      last = row
    end
    snapshot = Snapshot.new(@buffer.rope, @buffer.version, @id,
      @document.definition.name, @document.definition.lexer, first, last,
      {syntax: !!(@want_syntax && !syntax_complete?), selections: @selection_positions}.freeze)
    if (@job = @scheduler.submit(snapshot, prior_syntax: @syntax))
      @submitted_version, @submitted_generation = @buffer.version, @generation
    end
  end
  changed
rescue StandardError => error
  @job&.cancel
  @job = nil
  @failed, @dirty = true, false
  @analysis_error = error
  raise
end

#request(first_line: nil, last_line: nil, rows: nil, syntax: false) ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/canopus/language/background_analysis.rb', line 42

def request(first_line: nil, last_line: nil, rows: nil, syntax: false)
  return if @disabled || @disposed
  if rows
    unless rows.is_a?(Array) && rows.length.between?(1, ROW_LIMIT) && first_line.nil? && last_line.nil?
      raise ArgumentError, "language rows must contain 1..#{ROW_LIMIT} entries without a range"
    end
    rows.each { |row| validate_row(row) }
    rows = rows.uniq.sort.freeze
    @explicit_rows = true
  elsif first_line || last_line || !@rows
    first = first_line || @range&.first || 0
    last = last_line || @range&.last || first
    validate_row(first)
    validate_row(last)
    raise ArgumentError, "inverted language range" if last < first
    rows = (first..[last, first + ROW_LIMIT - 1].min).to_a.freeze
    @explicit_rows = false
  else
    rows = @rows
  end
  changed = rows != @rows || (syntax && !@want_syntax)
  if rows != @rows && !syntax_complete?
    @syntax = nil
    @syntax_rows.clear
  end
  @want_syntax ||= syntax
  if changed
    @job&.cancel
    @job = nil
    @generation += 1
    @failed = false
    @analysis_error = nil
  end
  @rows, @range = rows, [rows.first, rows.last].freeze
  @dirty = !@failed && missing_work?
  self
end

#selection_ranges(offsets) ⇒ Object 



94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/canopus/language/background_analysis.rb', line 94

def selection_ranges(offsets)
  unless offsets.is_a?(Array) && offsets.length.between?(1, SELECTION_LIMIT)
    raise ArgumentError, "selection offsets must contain 1..#{SELECTION_LIMIT} entries"
  end
  positions = offsets.each_with_index.map do |offset, index|
    point = @buffer.rope.point_at(offset)
    [index, point.row, point.column, offset].freeze
  end.freeze
  unless @selection_positions&.map(&:last) == offsets
    @job&.cancel
    @job = nil
    @selection_positions, @selection_results = positions, {}
    @generation += 1
    @failed = false
    @analysis_error = nil
    @dirty = missing_work?
  end
  return unless @selection_results.length == positions.length

  positions.map { |index,| @selection_results.fetch(index) }.freeze
end

#structure_regions ⇒ Object 



93
 
# File 'lib/canopus/language/background_analysis.rb', line 93

def structure_regions = (request(syntax: true); @syntax ? @syntax[:structure_regions] : EMPTY)

#syntax_complete? ⇒ Boolean

Returns:

  • (Boolean) 


83
 
# File 'lib/canopus/language/background_analysis.rb', line 83

def syntax_complete? = !!(@syntax && @syntax[:complete])

#syntax_ready? ⇒ Boolean

Returns:

  • (Boolean) 


82
 
# File 'lib/canopus/language/background_analysis.rb', line 82

def syntax_ready? = !!@syntax

#tokens_current?(row) ⇒ Boolean

Returns:

  • (Boolean) 


81
 
# File 'lib/canopus/language/background_analysis.rb', line 81

def tokens_current?(row) = @tokens.key?(row)

#tokens_for(row) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/canopus/language/background_analysis.rb', line 24

def tokens_for(row)
  validate_row(row)
  return @tokens[row] if @tokens.key?(row)
  unless @disabled || @disposed || @explicit_rows
    first, last = @range || [row, row]
    if row < first
      first, last = row, [last, row + ROW_LIMIT - 1].min
    elsif row > last
      first, last = [first, row - ROW_LIMIT + 1].max, row
    end
    request(first_line: first, last_line: last)
  end
  return @stale[row] if @stale.key?(row)
  return @provisional[row] if @provisional.key?(row)
  @provisional.shift if @provisional.length >= CACHE_ROWS
  @provisional[row] = [["Text", self.class.line(@buffer.rope, row)].freeze].freeze
end