Module: RegressionTest::CliExec

Defined in:
lib/regressiontest/cli_exec.rb

Overview

Regression test runner compares output in ./test/data/regression (by default). The convention is to have a file with names .ref (reference) and create .new

You can add an :ignore regex option which ignores lines in the comparson files matching a regex

:timeout sets the time out for calling a system command

:should_fail expects the system command to return a non-zero

Defined Under Namespace

Classes: FilePair

Class Method Summary collapse

Class Method Details

.compare_files(fn1, fn2, ignore = nil) ⇒ Object 



82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/regressiontest/cli_exec.rb', line 82

def CliExec::compare_files fn1, fn2, ignore = nil
  if not File.exist?(fn2)
    FileUtils::cp(fn1,fn2)
    true
  else
    cmd = "diff #{fn2} #{fn1}"
    $stderr.print cmd+"\n"
    return true if Kernel.system(cmd) == true
    $stderr.print "If it is correct, execute \"cp #{fn1} #{fn2}\", and run again"
    false
  end
end

.exec(command, testname, options = {}) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/regressiontest/cli_exec.rb', line 19

def CliExec::exec command, testname, options = {}
  # ---- Find .ref file
  fullname = DEFAULT_TESTDIR + "/" + testname 
  basefn = if File.exist?(testname+".ref") || File.exist?(testname+"-stderr.ref")
            testname 
          elsif File.exist?(fullname + ".ref") || File.exist?(fullname+"-stderr.ref")
            FileUtils.mkdir_p DEFAULT_TESTDIR
            fullname
          else
            raise "Can not find reference file for #{testname} - expected #{fullname}.ref"
          end
  std_out = FilePair.new(basefn + ".new", basefn + ".ref")
  std_err = FilePair.new(basefn + "-stderr.new", basefn + "-stderr.ref")
  files = [std_out,std_err]
  # ---- Create .new file
  cmd = command + " > #{std_out.outfn} 2>#{std_err.outfn}"
  $stderr.print cmd,"\n"
  exec_ret = nil
  if options[:timeout] && options[:timeout] > 0
    Timeout.timeout(options[:timeout]) do
      begin
        exec_ret = Kernel.system(cmd)
      rescue Timeout::Error
        $stderr.print cmd, " failed to finish in under #{options[:timeout]}\n"
        return false
      end
    end
  else
    exec_ret = Kernel.system(cmd)
  end
  expect_fail = (options[:should_fail] != nil)
  if !expect_fail and exec_ret==0
    $stderr.print cmd," returned an error\n"
    return false 
  end
  if expect_fail and exec_ret
    $stderr.print cmd," did not return an error\n"
    return false 
  end
  if options[:ignore]
    regex = options[:ignore]
    files.each do |f|
      outfn = f.outfn
      outfn1 = outfn + ".1"
      FileUtils.mv(outfn,outfn1)
      f1 = File.open(outfn1)
      f2 = File.open(outfn,"w")
      f1.each_line do | line |
        f2.print(line) if line !~ /#{regex}/
      end
      f1.close
      f2.close
      FileUtils::rm(outfn1)
    end
  end
  # ---- Compare the two files
  files.each do |f|
    next unless File.exist?(f.reffn)
    return false unless compare_files(f.outfn,f.reffn,options[:ignore])
  end
  return true
end