Class: Ryac::Pipeline::Concatenator

Inherits:
Object
  • Object
show all
Defined in:
lib/ryac/pipeline/concatenator.rb,
sig/ryac/pipeline/concatenator.rbs

Overview

Stage 2: File Concatenation Performs topological sort and concatenates files in dependency order

Defined Under Namespace

Classes: GraphSorter

Constant Summary collapse

REGISTRY_NAME =

The registry lazy regions register into and the loader that runs them: ordinary source, minified with everything else (both get short names), spelled so as not to collide with the program's own names.

Returns:

  • (String) 
'RYAC_LAZY'
LOADER_NAME =

Returns:

  • (String) 
'ryac_require'

Instance Method Summary collapse

Instance Method Details

#bundle_names(graph) ⇒ Array[String]

Names the program does not spell anywhere.

Parameters:

Returns:

  • (Array[String]) 


183
184
185
186
 
# File 'lib/ryac/pipeline/concatenator.rb', line 183

def bundle_names(graph)
  contents = graph.files.each_value.map(&:content)
  [REGISTRY_NAME, LOADER_NAME].map { |base| unused_name(base, contents) }
end

#call(graph) ⇒ ConcatenatedSource

Returns Ordered, concatenated source.

Parameters:

Returns:

Raises:



19
20
21
22
 
# File 'lib/ryac/pipeline/concatenator.rb', line 19

def call(graph)
  sorted_paths = topological_sort(graph)
  concatenate_files(graph, sorted_paths)
end

#collect_stdlib_requires(entry, stdlib_requires) ⇒ void

This method returns an undefined value.

Hoisting a require to the top of the output makes it run at load time. That is fine for one the file already ran at load time, but a require inside a method body runs only when the method is called and is often guarded — optcarrot loads stackprof only under --stackprof-mode, so hoisting it turns an optional dependency into a mandatory one.

Parameters:

  • entry (FileEntry)
  • stdlib_requires (Array[String]) 


234
235
236
237
238
239
240
 
# File 'lib/ryac/pipeline/concatenator.rb', line 234

def collect_stdlib_requires(entry, stdlib_requires)
  entry.require_nodes.each do |node|
    next unless node[:type] == :require_stdlib
    next if node[:in_method]
    stdlib_requires << node[:path]
  end
end

#common_root(paths) ⇒ String

The directory every bundled file sits under. Region keys and the respelled prefix of a dynamic require site are both relative to it, which is what makes a key and the string the site builds at runtime agree.

Parameters:

  • paths (Array[String])

Returns:

  • (String) 


168
169
170
171
172
 
# File 'lib/ryac/pipeline/concatenator.rb', line 168

def common_root(paths)
  root = File.dirname(paths.fetch(0))
  root = File.dirname(root) until root == '/' || paths.all? { |p| p.start_with?("#{root}/") }
  root
end

#concatenate_files(graph, sorted_paths) ⇒ ConcatenatedSource

Concatenate files in sorted order

Parameters:

Returns:



104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/ryac/pipeline/concatenator.rb', line 104

def concatenate_files(graph, sorted_paths)
  content_parts = [] #: Array[String]
  file_boundaries = [] #: Array[FileBoundary]
  stdlib_requires = [] #: Array[String]
  inlined = Set.new
  current_line = 1

  lazy_paths = graph.lazy_paths
  @root = common_root(graph.paths)
  @registry, @loader = lazy_paths.empty? ? [nil, nil] : bundle_names(graph)

  # Pre-clean all files: resolve in-class requires by inlining
  cleaned_cache = {} #: Hash[String, String]
  sorted_paths.each do |path|
    entry = graph[path]
    next unless entry
    collect_stdlib_requires(entry, stdlib_requires) unless entry.lazy
    cleaned_cache[path] = process_require_statements(entry, graph, inlined, cleaned_cache)
  end

  if (registry = @registry) && (loader = @loader)
    prelude = loader_prelude(registry, loader)
    content_parts << prelude
    current_line += prelude.count("\n") + 1
  end

  # Regions first: registering is all a region does at load, and it
  # has to be registered before any code that could ask for it runs.
  lazy, flat = sorted_paths.partition { |path| graph[path]&.lazy }
  (lazy + flat).each do |path|
    next if inlined.include?(path)
    entry = graph[path]
    next unless entry

    cleaned_content = cleaned_cache[path]
    cleaned_content = lazy_registration(path, cleaned_content) if entry.lazy
    lines = cleaned_content.count("\n") + 1

    file_boundaries << FileBoundary.new(
      path: path,
      start_line: current_line,
      end_line: current_line + lines - 1
    )

    content_parts << cleaned_content
    current_line += lines
  end

  original_size = graph.files.values.sum { |f| f.content.bytesize }

  ConcatenatedSource.new(
    content: content_parts.join("\n"),
    file_boundaries: file_boundaries,
    original_size: original_size,
    stdlib_requires: stdlib_requires.uniq,
    rbs_files: graph.rbs_files,
    lazy_files: lazy_paths
  )
end

#extract_cycle_from_graph(graph) ⇒ Array[String]

Extract cycle path from graph for error reporting

Parameters:

Returns:

  • (Array[String]) 


40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/ryac/pipeline/concatenator.rb', line 40

def extract_cycle_from_graph(graph)
  visited = Set.new
  rec_stack = Set.new
  path = [] #: Array[String]

  graph.paths.each do |start|
    if find_cycle_dfs(graph, start, visited, rec_stack, path)
      return path
    end
  end

  # Fallback: return all paths
  graph.paths
end

#find_cycle_dfs(graph, node, visited, rec_stack, path) ⇒ Boolean

Parameters:

  • graph (DependencyGraph)
  • node (String)
  • visited (Set[String])
  • rec_stack (Set[String])
  • path (Array[String])

Returns:

  • (Boolean) 


55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/ryac/pipeline/concatenator.rb', line 55

def find_cycle_dfs(graph, node, visited, rec_stack, path)
  return false if visited.include?(node)

  visited.add(node)
  rec_stack.add(node)
  path << node

  entry = graph[node]
  deps = entry&.dependencies
  if deps
    deps.each do |dep|
      if rec_stack.include?(dep)
        path << dep
        return true
      end

      if find_cycle_dfs(graph, dep, visited, rec_stack, path)
        return true
      end
    end
  end

  rec_stack.delete(node)
  path.pop
  false
end

#lazy_key(path) ⇒ String

Parameters:

  • path (String)

Returns:

  • (String) 


178
179
180
 
# File 'lib/ryac/pipeline/concatenator.rb', line 178

def lazy_key(path)
  relative_to_root(path).delete_suffix('.rb')
end

#lazy_registration(path, content) ⇒ String

The file as a region (see LazyRegions): its own line for the closing brace, so a trailing comment cannot swallow it.

Parameters:

  • path (String)
  • content (String)

Returns:

  • (String) 


225
226
227
 
# File 'lib/ryac/pipeline/concatenator.rb', line 225

def lazy_registration(path, content)
  "#{@registry}[#{lazy_key(path).dump}] = -> {\n#{content}\n}"
end

#lazy_site_call(bytes, node) ⇒ String

require_relative "driver/#{name}_#{type}" becomes ryac_require("optcarrot/driver/#{name}_#{type}"): the literal keeps its interpolation, its static prefix is respelled relative to the bundle root — the spelling the regions were registered under — and a literal ".rb" tail goes, as keys carry no extension. The bytes are read before this node's own edit, so every offset is still original.

Parameters:

  • bytes (String)
  • node (require_node_info)

Returns:

  • (String) 


317
318
319
320
321
322
323
324
325
326
327
 
# File 'lib/ryac/pipeline/concatenator.rb', line 317

def lazy_site_call(bytes, node)
  site = node.fetch(:lazy)
  arg = bytes[site[:arg_start_offset]...site[:arg_end_offset]] #: String
  base = site[:arg_start_offset]
  if (suffix_start = site[:suffix_start_offset])
    arg[(suffix_start - base), 3] = ''
  end
  dir = site[:dir]
  arg[(site[:prefix_start_offset] - base), site[:prefix_length]] = (dir == @root ? '' : "#{relative_to_root(dir)}/").b
  "#{@loader}(#{arg})"
end

#line_based_processing(content, require_nodes) ⇒ String

Parameters:

  • content (String)
  • require_nodes (Array[require_node_info])

Returns:

  • (String) 


329
330
331
332
333
334
335
336
 
# File 'lib/ryac/pipeline/concatenator.rb', line 329

def line_based_processing(content, require_nodes)
  lines = content.lines
  lines_to_remove = Set.new
  require_nodes.each { |node| lines_to_remove.add(node[:line] - 1) }
  lines.each_with_index.map do |line, idx|
    lines_to_remove.include?(idx) ? '' : line.chomp
  end.join("\n")
end

#loader_prelude(registry, loader) ⇒ String

The loader keeps Ruby's require contract for the regions it owns: a region runs once and answers true, a repeat answers false, a region whose run raised (its require "ffi" had no ffi) is restored so a later require can try it again, and a path no region was registered under falls through to a real require_relative — relative to the bundle, the one file all this code now lives in.

Parameters:

  • registry (String)
  • loader (String)

Returns:

  • (String) 


204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
 
# File 'lib/ryac/pipeline/concatenator.rb', line 204

def loader_prelude(registry, loader)
  "    \#{registry} = {}\n    def \#{loader}(path)\n      return require_relative(path) unless \#{registry}.key?(path)\n      body = \#{registry}[path]\n      return false unless body\n      \#{registry}[path] = false\n      begin\n        body.call\n      rescue Exception\n        \#{registry}[path] = body\n        raise\n      end\n      true\n    end\n  RUBY\nend\n"

#offset_based_processing(content, nodes, graph, in_class_deps, inlined, cleaned_cache, lazy_entry) ⇒ String

Parameters:

  • content (String)
  • nodes (Array[require_node_info])
  • graph (DependencyGraph)
  • in_class_deps (Set[String])
  • inlined (Set[String])
  • cleaned_cache (Hash[String, String])
  • lazy_entry (Boolean)

Returns:

  • (String) 


258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
 
# File 'lib/ryac/pipeline/concatenator.rb', line 258

def offset_based_processing(content, nodes, graph, in_class_deps, inlined, cleaned_cache, lazy_entry)
  sorted_nodes = nodes.sort_by { |n| n[:start_offset] }.reverse
  # Prism offsets are byte offsets: splice on bytes, or any multibyte
  # character before a require shifts every slice after it.
  result = content.b
  sorted_nodes.each do |node|
    start_pos = node[:start_offset]
    end_pos = start_pos + node[:length]

    if node[:type] == :require_lazy
      result[start_pos...end_pos] = lazy_site_call(result, node).b
      next
    end

    dep_path = resolve_node_path(node, graph)
    # A require that names a region asks the loader for it, whether
    # the site is a static sibling require inside another region or
    # an autoload — which then loads at that point, as it does for a
    # flat file.
    if dep_path && graph[dep_path]&.lazy
      result[start_pos...end_pos] = "#{@loader}(#{lazy_key(dep_path).dump})".b
      next
    end

    if node[:in_class] && !node[:in_method] && node[:type] != :require_stdlib
      if dep_path && graph[dep_path]
        # the `graph[dep_path]` check above guarantees the entry exists
        dep_content = cleaned_cache[dep_path] || graph[dep_path].content # steep:ignore NoMethod
        stripped = strip_outer_nesting(dep_content)
        # Only consume trailing semicolons (not newlines) for inline
        while end_pos < result.size && result[end_pos] == ';'
          end_pos += 1
        end
        result[start_pos...end_pos] = stripped.b
        inlined.add(dep_path)
        next
      end
    end

    # An in-method stdlib require is not hoisted, so it has to stay where
    # it is — deleting it here would drop the require altogether. Nothing
    # in a region is hoisted: its requires run when the region does.
    next if node[:type] == :require_stdlib && (node[:in_method] || lazy_entry)

    # For removal: consume trailing semicolons and newlines
    while end_pos < result.size && (result[end_pos] == ';' || result[end_pos] == "\n")
      end_pos += 1
    end
    result[start_pos...end_pos] = ''
  end
  result.force_encoding(content.encoding)
end

#process_require_statements(entry, graph, inlined, cleaned_cache) ⇒ String

Process require statements: remove top-level requires, inline in-class requires

Parameters:

Returns:

  • (String) 


243
244
245
246
247
248
249
250
251
252
253
254
255
256
 
# File 'lib/ryac/pipeline/concatenator.rb', line 243

def process_require_statements(entry, graph, inlined, cleaned_cache)
  content = entry.content
  require_nodes = entry.require_nodes
  return content if require_nodes.empty?

  in_class_deps = entry.in_class_dependencies.to_set
  nodes_with_offsets = require_nodes.select { |n| n[:start_offset] }

  if nodes_with_offsets.size == require_nodes.size
    offset_based_processing(content, nodes_with_offsets, graph, in_class_deps, inlined, cleaned_cache, entry.lazy)
  else
    line_based_processing(content, require_nodes)
  end
end

#relative_to_root(path) ⇒ String

Parameters:

  • path (String)

Returns:

  • (String) 


174
175
176
 
# File 'lib/ryac/pipeline/concatenator.rb', line 174

def relative_to_root(path)
  path.delete_prefix(@root == '/' ? '/' : "#{@root}/")
end

#resolve_node_path(node, graph) ⇒ String?

Parameters:

Returns:

  • (String, nil) 


338
339
340
341
342
 
# File 'lib/ryac/pipeline/concatenator.rb', line 338

def resolve_node_path(node, graph)
  resolved = node[:resolved_path]
  return nil unless resolved
  graph.files.key?(resolved) ? resolved : nil
end

#strip_outer_nesting(content) ⇒ String

Strip outer module/class nesting from a file so it can be inlined inside the parent's class body. Peels single-child module/class layers until reaching the innermost new scope definition.

Parameters:

  • content (String)

Returns:

  • (String) 


347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
 
# File 'lib/ryac/pipeline/concatenator.rb', line 347

def strip_outer_nesting(content)
  ast = Prism.parse(content).value
  node = ast.statements
  while node.is_a?(Prism::StatementsNode) && node.body.size == 1
    child = node.body.first
    break unless child.is_a?(Prism::ModuleNode) || child.is_a?(Prism::ClassNode)
    inner_body = child.body
    if inner_body.is_a?(Prism::StatementsNode) && inner_body.body.size == 1
      inner_child = inner_body.body.first
      if inner_child.is_a?(Prism::ModuleNode) || inner_child.is_a?(Prism::ClassNode)
        node = inner_body
        next
      end
    end
    break
  end
  node.slice
end

#topological_sort(graph) ⇒ Array<String>

Perform topological sort using Ruby's TSort library

Parameters:

Returns:

  • (Array<String>) —

    Paths in dependency order (dependencies first)



28
29
30
31
32
33
34
35
36
37
 
# File 'lib/ryac/pipeline/concatenator.rb', line 28

def topological_sort(graph)
  sorter = GraphSorter.new(graph)
  begin
    sorter.tsort
  rescue TSort::Cyclic
    # Extract cycle from error message
    cycle = extract_cycle_from_graph(graph)
    raise CircularDependencyError.new(cycle)
  end
end

#unused_name(base, contents) ⇒ String

Parameters:

  • base (String)
  • contents (Array[String])

Returns:

  • (String) 


188
189
190
191
192
193
194
195
196
 
# File 'lib/ryac/pipeline/concatenator.rb', line 188

def unused_name(base, contents)
  name = base
  suffix = 0
  while contents.any? { |c| c.match?(/\b#{Regexp.escape(name)}\b/) }
    suffix += 1
    name = "#{base}_#{suffix}"
  end
  name
end