Module: Autobuild::Subprocess

Defined in:
lib/autobuild/subprocess.rb

Overview

rubocop:disable Style/ClassAndModuleChildren

Defined Under Namespace

Classes: Failed

Class Method Summary collapse

Class Method Details

.compute_log_path(target_name, phase, logdir) ⇒ Object 



407
408
409
410
 
# File 'lib/autobuild/subprocess.rb', line 407

def self.compute_log_path(target_name, phase, logdir)
    File.join(logdir, "#{target_name.gsub(/:/, '_')}-"\
        "#{phase.to_s.gsub(/:/, '_')}.log")
end

.feed_input(input_streams, out_r, stdin_w) ⇒ Object 



495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
 
# File 'lib/autobuild/subprocess.rb', line 495

def self.feed_input(input_streams, out_r, stdin_w)
    readbuffer = StringIO.new
    input_streams.each do |instream|
        instream.each_line do |line|
            # Read the process output to avoid having it block on a full pipe
            begin
                loop do
                    readbuffer.write(out_r.read_nonblock(1024))
                end
            rescue IO::WaitReadable # rubocop:disable Lint/SuppressedException
            end

            stdin_w.write(line)
        end
    end
    readbuffer
rescue Errno::ENOENT => e
    raise Failed.new(nil, false),
          "cannot open input files: #{e.message}", retry: false
end

.handle_exit_status(status, command) ⇒ Object 



381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
 
# File 'lib/autobuild/subprocess.rb', line 381

def self.handle_exit_status(status, command)
    return if status.exitstatus == 0

    if status.termsig == 2 # SIGINT == 2
        raise Interrupt, "subcommand #{command.join(' ')} interrupted"
    end

    if status.termsig
        raise Failed.new(status.exitstatus, nil),
              "'#{command.join(' ')}' terminated by signal #{status.termsig}"
    else
        raise Failed.new(status.exitstatus, nil),
              "'#{command.join(' ')}' returned status #{status.exitstatus}"
    end
end

.logfile_header(logfile, command, env) ⇒ Object 



428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
 
# File 'lib/autobuild/subprocess.rb', line 428

def self.logfile_header(logfile, command, env)
    logfile.puts if Autobuild.keep_oldlogs
    logfile.puts
    logfile.puts "#{Time.now}: running"
    logfile.puts "    #{command.join(' ')}"
    logfile.puts "with environment:"
    env.keys.sort.each do |key|
        if (value = env[key])
            logfile.puts "  '#{key}'='#{value}'"
        end
    end
    logfile.puts
    logfile.puts "#{Time.now}: running"
    logfile.puts "    #{command.join(' ')}"
    logfile.flush
    logfile.sync = true
end

.open_logfile(logname, &block) ⇒ Object 



412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
 
# File 'lib/autobuild/subprocess.rb', line 412

def self.open_logfile(logname, &block)
    open_flag = if Autobuild.keep_oldlogs then 'a'
                elsif Autobuild.registered_logfile?(logname) then 'a'
                else
                    'w'
                end
    open_flag << ":BINARY"

    unless File.directory?(File.dirname(logname))
        FileUtils.mkdir_p File.dirname(logname)
    end

    Autobuild.register_logfile(logname)
    File.open(logname, open_flag, &block)
end

.outpipe_each_line(out_r) ⇒ Object 



446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
 
# File 'lib/autobuild/subprocess.rb', line 446

def self.outpipe_each_line(out_r)
    buffer = +""
    while (data = out_r.readpartial(1024))
        buffer << data
        scanner = StringScanner.new(buffer)
        while (line = scanner.scan_until(/\n/))
            yield line
        end
        buffer = scanner.rest.dup
    end
rescue EOFError
    scanner = StringScanner.new(buffer)
    while (line = scanner.scan_until(/\n/))
        yield line
    end
    yield scanner.rest unless scanner.rest.empty?
end

.process_output(out_r, logfile, transparent_prefix, encoding, &filter) ⇒ Object 



464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
 
# File 'lib/autobuild/subprocess.rb', line 464

def self.process_output(
    out_r, logfile, transparent_prefix, encoding, &filter
)
    subcommand_output = []
    outpipe_each_line(out_r) do |line|
        line.force_encoding(encoding)
        line = line.chomp
        subcommand_output << line

        logfile.puts line

        if Autobuild.verbose || transparent_mode?
            STDOUT.puts "#{transparent_prefix}#{line}"
        elsif filter
            # Do not yield
            # would mix the progress output with the actual command
            # output. Assume that if the user wants the command output,
            # the autobuild progress output is unnecessary
            filter.call(line)
        end
    end
    subcommand_output
end

.run(target, phase, *command) {|line| ... } ⇒ String

Run a subcommand and return its standard output

The command’s standard and error outputs, as well as the full command line and an environment dump are saved in a log file in either the valure returned by target#logdir, or Autobuild.logdir if the target does not respond to #logdir.

The subprocess priority is controlled by Autobuild.nice

Parameters:

  • target (String, (#name,#logdir,#working_directory)) —

    the target we run the subcommand for. In general, it will be a Package object (run from Package#run)

  • phase (String) —

    in which build phase this subcommand is executed

  • the (Array<String>) —

    command itself

  • options (Hash)

Yield Parameters:

  • line (String) —

    if a block is given, each output line from the command’s standard output are yield to it. This is meant for progress display, and is disabled if Autobuild.verbose is set.

Returns:

  • (String) —

    the command standard output



205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
 
# File 'lib/autobuild/subprocess.rb', line 205

def self.run(target, phase, *command, &output_filter)
    STDOUT.sync = true

    input_streams = []
    options = {
        retry: false, encoding: 'BINARY',
        env: ENV.to_hash, env_inherit: true
    }

    if command.last.kind_of?(Hash)
        options = command.pop
        options = Kernel.validate_options(
            options,
            input: nil, working_directory: nil, retry: false,
            input_streams: [],
            env: ENV.to_hash,
            env_inherit: true,
            encoding: 'BINARY'
        )

        input_streams << File.open(options[:input]) if options[:input]
        input_streams.concat(options[:input_streams]) if options[:input_streams]
    end

    start_time = Time.now

    # Filter nil and empty? in command
    command.reject! { |o| o.nil? || (o.respond_to?(:empty?) && o.empty?) }
    command.collect!(&:to_s)

    target_name, target_type = target_argument_to_name_and_type(target)

    logdir = if target.respond_to?(:logdir)
                 target.logdir
             else
                 Autobuild.logdir
             end

    if target.respond_to?(:working_directory)
        options[:working_directory] ||= target.working_directory
    end

    env = options[:env].dup
    if options[:env_inherit]
        ENV.each do |k, v|
            env[k] = v unless env.key?(k)
        end
    end

    if Autobuild.windows?
        windows_support(options, command)
        return
    end

    logname = compute_log_path(target_name, phase, logdir)

    status, subcommand_output = open_logfile(logname) do |logfile|
        logfile_header(logfile, command, env)

        if Autobuild.verbose
            Autobuild.message "#{target_name}: running #{command.join(' ')}\n"\
                "    (output goes to #{logname})"
        end

        unless input_streams.empty?
            stdin_r, stdin_w = IO.pipe # to feed subprocess stdin
        end

        out_r, out_w = IO.pipe
        out_r.sync = true
        out_w.sync = true

        logfile.puts "Spawning"
        stdin_redir = { :in => stdin_r } if stdin_r
        begin
            pid = spawn(
                env, *command,
                {
                    :chdir => options[:working_directory] || Dir.pwd,
                    :close_others => false,
                    %I[err out] => out_w
                }.merge(stdin_redir || {})
            )
            logfile.puts "Spawned, PID=#{pid}"
        rescue Errno::ENOENT
            raise Failed.new(nil, false), "command '#{command.first}' not found"
        end

        if Autobuild.nice
            Process.setpriority(Process::PRIO_PROCESS, pid, Autobuild.nice)
        end

        # Feed the input
        unless input_streams.empty?
            logfile.puts "Feeding STDIN"
            stdin_r.close
            readbuffer = feed_input(input_streams, out_r, stdin_w)
            stdin_w.close
        end

        # If the caller asked for process output, provide it to him
        # line-by-line.
        out_w.close

        unless input_streams.empty?
            readbuffer.write(out_r.read)
            readbuffer.seek(0)
            out_r.close
            out_r = readbuffer
        end

        transparent_prefix =
            transparent_output_prefix(target_name, phase, target_type)
        logfile.puts "Processing command output"
        subcommand_output = process_output(
            out_r, logfile, transparent_prefix, options[:encoding], &output_filter
        )
        out_r.close

        logfile.puts "Waiting for #{pid} to finish"
        _, childstatus = Process.wait2(pid)
        logfile.puts "Exit: #{childstatus}"
        [childstatus, subcommand_output]
    end

    handle_exit_status(status, command)
    update_stats(target, phase, start_time)

    subcommand_output
rescue Failed => e
    error = Autobuild::SubcommandFailed.new(
        target, command.join(" "), logname, e.status, subcommand_output || []
    )
    error.retry = if e.retry?.nil? then options[:retry]
                  else
                      e.retry?
                  end
    error.phase = phase
    raise error, e.message
end

.target_argument_to_name(target) ⇒ Object 



367
368
369
370
371
372
373
 
# File 'lib/autobuild/subprocess.rb', line 367

def self.target_argument_to_name(target)
    if target.respond_to?(:name)
        target.name
    else
        target.to_str
    end
end

.target_argument_to_name_and_type(target) ⇒ Object 



359
360
361
362
363
364
365
 
# File 'lib/autobuild/subprocess.rb', line 359

def self.target_argument_to_name_and_type(target)
    if target.respond_to?(:name)
        [target.name, target.class]
    else
        [target.to_str, nil]
    end
end

.target_argument_to_type(target, type) ⇒ Object 



375
376
377
378
379
 
# File 'lib/autobuild/subprocess.rb', line 375

def self.target_argument_to_type(target, type)
    return type if type

    target.class if target.respond_to?(:name)
end

.transparent_mode=(flag) ⇒ Object 



168
169
170
 
# File 'lib/autobuild/subprocess.rb', line 168

def self.transparent_mode=(flag)
    @transparent_mode = flag
end

.transparent_mode? ⇒ Boolean

Returns:

  • (Boolean) 


164
165
166
 
# File 'lib/autobuild/subprocess.rb', line 164

def self.transparent_mode?
    @transparent_mode
end

.transparent_output_prefix(target_name, phase, target_type) ⇒ Object 



488
489
490
491
492
493
 
# File 'lib/autobuild/subprocess.rb', line 488

def self.transparent_output_prefix(target_name, phase, target_type)
    prefix = "#{target_name}:#{phase}: "
    return prefix unless target_type

    "#{target_type}:#{prefix}"
end

.update_stats(target, phase, start_time) ⇒ Object 



346
347
348
349
350
351
352
353
354
355
356
357
 
# File 'lib/autobuild/subprocess.rb', line 346

def self.update_stats(target, phase, start_time)
    duration = Time.now - start_time
    target_name, = target_argument_to_name_and_type(target)
    Autobuild.add_stat(target, phase, duration)
    FileUtils.mkdir_p(Autobuild.logdir)
    File.open(File.join(Autobuild.logdir, "stats.log"), 'a') do |io|
        formatted_msec = format('%.03i', start_time.tv_usec / 1000)
        formatted_time = "#{start_time.strftime('%F %H:%M:%S')}.#{formatted_msec}"
        io.puts "#{formatted_time} #{target_name} #{phase} #{duration}"
    end
    target.add_stat(phase, duration) if target.respond_to?(:add_stat)
end

.windows_support(options, command) ⇒ Object 



397
398
399
400
401
402
403
404
405
 
# File 'lib/autobuild/subprocess.rb', line 397

def self.windows_support(options, command)
    Dir.chdir(options[:working_directory]) do
        unless system(*command)
            exit_code = $CHILD_STATUS.exitstatus
            raise Failed.new(exit_code, nil),
                  "'#{command.join(' ')}' returned status #{exit_code}"
        end
    end
end