Class: TestCase

Inherits:
MiniTest::Test
  • Object
show all
Includes:
Helpers, LegacyLogging, PerfLib, TmcHelpers, TmcUtils
Defined in:
lib/test_case/test_case.rb

Overview

The TestCase class manages test flow

Direct Known Subclasses

FunctionalTestCase, PerformanceTestCase

Constant Summary collapse

@@not_valid_for = 
{}
@@valid_for = 
[]
@@prereqs_for_test = 
[]
@@prereqs_for_dut = 
[]

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from TmcHelpers

#aws_config, #aws_create_s3_bucket, #aws_delete_s3, #aws_delete_s3_bucket, #aws_read_s3, #aws_upload_s3, #aws_write_s3, #get_clean_ocr_text, #get_readable_size, #get_readable_time, #get_string_similarity, #get_substring_similarity, #is_rating?, #is_year?, #jsonify, #pick_random, #rubify, #send_webex_alert, #to_camel_case, #to_pascal_case, #to_snake_case, #twb_case_exists?, #twb_get_case_instance_id, #twb_get_suite_instance_id, #twb_post_screenshot, #twb_post_screenshot!, #twb_post_suite, #twb_post_testcase, #twb_post_teststep, #twb_post_teststep!, #twb_prep_s3_screenshot, #twb_suite_exists?, #until_condition, #until_equals, #until_includes, #until_not_empty, #until_not_equals, #until_not_includes, #until_not_nil, #until_same, #web_delete, #web_get, #web_post, #web_put

Methods included from HttpHelper

#web_request

Methods included from CsvHelper

#csv_parse, #csv_read

Methods included from SshHelper

#ssh_to

Methods included from EmailHelper

#send_email, #send_email_setup

Methods included from SnmpHelper

#snmp_get, #snmp_set

Constructor Details

#initialize(id, device_ids, iteration, host, job, log_file, log_level, data: nil, auth: nil, workspace: nil) ⇒ TestCase

Public: Initializes a new TestCase. NOT FOR USE IN TESTS.

Returns nothing.



86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
 
# File 'lib/test_case/test_case.rb', line 86

def initialize(id, device_ids, iteration, host, job, log_file, log_level, data: nil, auth: nil, workspace: nil)
  super(nil)
  @status = TestStatus.new(self, nil)
  # Set attributes from parameters

  @id = id
  @device_ids = device_ids || [] # needed for syntax eval

  @primary_device_id = @device_ids.empty? ? nil : @device_ids.first
  @identity = job
  @iteration = iteration
  @host = host
  @job = job
  @workspace = workspace
  # Set remaining attributes

  @type = nil
  @start_time = nil
  @stop_time = nil
  @teardowns = []
  @steps = []
  @errors = []
  @current_step = nil
  @data = nil
  @duts = {}
  @loaded_libs = false
  @recording_video_id = nil
  @video_recording_ids = {}
  @audio_recording_ids = {}
  @locked_resources = {}
  @reported_results = false
  @test_steps_ignore_errors = false
  @ignore_errors = false
  @no_log_trace = false
  # Set these attributes last as they may depend on others

  # Create base logger

  base_logger = TmcLogger.new(log_file, log_level)
  @loggers = {}
  if device?
    # Create DUT logger for each device

    @device_ids.each_with_index do |d, i|
      @loggers[d] = TmcDutLogger.new(i + 1, base_logger, self)
    end
  else
    # Create a non-device logger

    @loggers[0] = TmcDutLogger.new(nil, base_logger, self)
  end
  @logger = @loggers.values.first
  @test_data = TestData.new(self, job: @job, data: data)
  @ssh = TmcSsh.new(self)
  # Anything that logs should go here

  unless auth.nil?
    # Log in using base-64 auth

    resp = tmc_post('/api/login', json: { value: auth })
    @identity = resp['guid']
  end
  @ignore_interrupt = false
  @canceled = false
  trap_signals
end

Instance Attribute Details

#device_ids=(value) ⇒ Object (writeonly)

Sets the attribute device_ids

Parameters:

  • value —

    the value to set the attribute device_ids to.



81
82
83
 
# File 'lib/test_case/test_case.rb', line 81

def device_ids=(value)
  @device_ids = value
end

#host ⇒ Object (readonly)

Returns the value of attribute host.



80
81
82
 
# File 'lib/test_case/test_case.rb', line 80

def host
  @host
end

#id ⇒ Object (readonly)

Returns the value of attribute id.



80
81
82
 
# File 'lib/test_case/test_case.rb', line 80

def id
  @id
end

#identity ⇒ Object (readonly)

Returns the value of attribute identity.



80
81
82
 
# File 'lib/test_case/test_case.rb', line 80

def identity
  @identity
end

#iteration ⇒ Object (readonly)

Returns the value of attribute iteration.



80
81
82
 
# File 'lib/test_case/test_case.rb', line 80

def iteration
  @iteration
end

#job ⇒ Object (readonly)

Returns the value of attribute job.



80
81
82
 
# File 'lib/test_case/test_case.rb', line 80

def job
  @job
end

#logger ⇒ Object (readonly)

Returns the value of attribute logger.



80
81
82
 
# File 'lib/test_case/test_case.rb', line 80

def logger
  @logger
end

#reported_result=(value) ⇒ Object (writeonly)

Sets the attribute reported_result

Parameters:

  • value —

    the value to set the attribute reported_result to.



81
82
83
 
# File 'lib/test_case/test_case.rb', line 81

def reported_result=(value)
  @reported_result = value
end

#ssh ⇒ Object (readonly)

Returns the value of attribute ssh.



80
81
82
 
# File 'lib/test_case/test_case.rb', line 80

def ssh
  @ssh
end

#start_time ⇒ Object (readonly)

Returns the value of attribute start_time.



80
81
82
 
# File 'lib/test_case/test_case.rb', line 80

def start_time
  @start_time
end

#stop_time ⇒ Object (readonly)

Returns the value of attribute stop_time.



80
81
82
 
# File 'lib/test_case/test_case.rb', line 80

def stop_time
  @stop_time
end

Class Method Details

.not_valid_for(*device_types) ⇒ Object

Public: Assigns device types for which the test case is not valid.

device_types - Splat Array of device types for which the test case is not valid.

Returns nothing.



43
44
45
46
47
48
49
50
51
 
# File 'lib/test_case/test_case.rb', line 43

def self.not_valid_for(*device_types)
  if device_types.empty?
    nil
  elsif device_types.first.is_a?(Hash)
    @@not_valid_for.update(device_types.first)
  else
    device_types.each { |d| @@not_valid_for[d] = nil }
  end
end

.prereqs_for_dut(*prereqs_for_dut) ⇒ Object

Public: Assigns prerequisites for the DUT.

prereqs_for_dut - Splat Array of Symbol prereqs required for the DUT.

Returns nothing.



76
77
78
 
# File 'lib/test_case/test_case.rb', line 76

def self.prereqs_for_dut(*prereqs_for_dut)
  @@prereqs_for_dut = prereqs_for_dut
end

.prereqs_for_test(*prereqs_for_test) ⇒ Object

Public: Assigns prerequisites for the test.

prereqs_for_test - Splat Array of Symbol prereqs required for the test.

Returns nothing.



67
68
69
 
# File 'lib/test_case/test_case.rb', line 67

def self.prereqs_for_test(*prereqs_for_test)
  @@prereqs_for_test = prereqs_for_test
end

.valid_for(*device_types) ⇒ Object

Public: Assigns device types for which the test case is valid.

device_types - Splat Array of device types for which the test case is valid.

Returns nothing.



58
59
60
 
# File 'lib/test_case/test_case.rb', line 58

def self.valid_for(*device_types)
  @@valid_for += device_types
end

Instance Method Details

#`(_) ⇒ Object

Public: Block probes.



152
153
154
 
# File 'lib/test_case/test_case.rb', line 152

def `(_)
  probe_fail('`')
end

#add_teardown(msg, *args, &block) ⇒ Object

Public: Adds a teardown with the given method and arguments.

msg - String message to log when the teardown runs. args - Array of parameters to send (default: []). block - Block method the teardown should run.

Returns nothing.



472
473
474
475
 
# File 'lib/test_case/test_case.rb', line 472

def add_teardown(msg, *args, &block)
  logger.debug("Adding teardown: #{method_to_str(block)}")
  @teardowns << { message: msg, method: block, args: args }
end

#browser(browser: :firefox) ⇒ Object

Public: Gets a web browser.

browser - Symbol browser to launch, can be :firefox or :chrome (default: :chrome).

Returns the browser Object.



557
558
559
560
561
562
563
 
# File 'lib/test_case/test_case.rb', line 557

def browser(browser: :firefox)
  if @browser.nil? || @browser.send(:closed?)
    require './lib/platform/web/web'
    @browser = Web.new(browser, self)
  end
  @browser
end

#builtin_sleep ⇒ Object

Public: Sleeps. TODO: We really should not be clobbering built-ins like this.

delay - Integer or Range of milliseconds to sleep.

If a Range, will sleep a random number of milliseconds within the Range.

Returns nothing.



292
 
# File 'lib/test_case/test_case.rb', line 292

alias builtin_sleep sleep

#data ⇒ Object

Public: Gets the data service.

Returns the <data> instance.



421
422
423
424
 
# File 'lib/test_case/test_case.rb', line 421

def data
  @data = DataServices.new(@logger) if @data.nil?
  @data
end

#device? ⇒ Boolean

Public: Indicates whether the test has a device.

Returns a Boolean indicating if the test has a device.

Returns:

  • (Boolean) 


387
388
389
 
# File 'lib/test_case/test_case.rb', line 387

def device?
  !@device_ids.empty?
end

#dut ⇒ Object

Public: Gets the device under test, initializing it first if necessary.

Returns the dut.



527
528
529
 
# File 'lib/test_case/test_case.rb', line 527

def dut
  dut1
end

#dut1 ⇒ Object

Public: Gets the 1st device under test, initializing it first if necessary.

Returns the 1st dut.



534
535
536
 
# File 'lib/test_case/test_case.rb', line 534

def dut1
  get_dut(1)
end

#dut2 ⇒ Object

Public: Gets the 2nd device under test, initializing it first if necessary.

Returns the 2nd dut.



541
542
543
 
# File 'lib/test_case/test_case.rb', line 541

def dut2
  get_dut(2)
end

#dut3 ⇒ Object

Public: Gets the 3rd device under test, initializing it first if necessary.

Returns the 3rd dut.



548
549
550
 
# File 'lib/test_case/test_case.rb', line 548

def dut3
  get_dut(3)
end

#error(reason = 'Unknown error') ⇒ Object

Public: Quits the test.

reason - String message describing the reason for error.

Returns nothing.



461
462
463
 
# File 'lib/test_case/test_case.rb', line 461

def error(reason = 'Unknown error')
  raise reason
end

#errors ⇒ Object

Public: Gets test errors.

Returns an Array of errors in format:

[{:message => 'oh no!', :file => 'foo.rb', :line => 22, :method => 'foo', :line_in_method => 3, :step_id => 1, :step_name => 'Baz'}]



214
215
216
 
# File 'lib/test_case/test_case.rb', line 214

def errors
  @errors.map {|e| e.to_h}
end

#exec(*_c) ⇒ Object

Public: Block probes.



157
158
159
 
# File 'lib/test_case/test_case.rb', line 157

def exec(*_c)
  probe_fail('exec')
end

#fail(reason = 'Unknown failure') ⇒ Object

Public: Fails the test or step.

reason - String message describing the reason for failure.

Returns nothing.



431
432
433
434
435
436
437
438
439
440
 
# File 'lib/test_case/test_case.rb', line 431

def fail(reason = 'Unknown failure')
  sword_attach_screenshot reason
  backtrace = caller
  if @current_step.nil?
    @status = TestStatus.new(self, TestStatus::FAILED, reason: reason, backtrace: backtrace)
    raise TestException.new
  else
    @current_step.fail(reason, backtrace: backtrace)
  end
end

#get_time_delta(t1, t2) ⇒ Object Also known as: time_diff_milli

Public: Gets delta between the specified times.

t1 - Time, DateTime, Integer, Float, or String T1 time value. t2 - Time, DateTime, Integer, Float, or String T2 time value.

Returns the Float delta in milliseconds.



351
352
353
 
# File 'lib/test_case/test_case.rb', line 351

def get_time_delta(t1, t2)
  (to_time(t2) - to_time(t1)) * 1000.0 # convert to ms

end

#ignore_errors(val) ⇒ Object

Public: Sets test case errors behavior.

val - A Boolean indicating whether the test case should ignore errors in determining the result,

when test steps are also configured to ignore errors.

Returns nothing.



206
207
208
 
# File 'lib/test_case/test_case.rb', line 206

def ignore_errors(val)
  @ignore_errors = val
end

#ignore_errors? ⇒ Boolean

Public: Gets test case errors behavior.

Returns a Boolean indicating the test case ignores errors in determining the result, when test steps are also configured to ignore errors.

Returns:

  • (Boolean) 


196
197
198
 
# File 'lib/test_case/test_case.rb', line 196

def ignore_errors?
  @ignore_errors
end

#job_name ⇒ Object

Public: Gets job name.

Returns the String name of the current job.



513
514
515
 
# File 'lib/test_case/test_case.rb', line 513

def job_name
  job_status['name']
end

#job_start_time ⇒ Object

Public: Gets job start time.

Returns the String start time of the current job in ISO 8601 format, local to the TMC, with a timezone offset.



520
521
522
 
# File 'lib/test_case/test_case.rb', line 520

def job_start_time
  job_status['startedAt'].to_time
end

#local_utc_offset ⇒ Object

Public: Gets the local UTC offset.

Returns the local time offset in Integer milliseconds.



568
569
570
571
572
 
# File 'lib/test_case/test_case.rb', line 568

def local_utc_offset
  local_offset_str = Time.now.getlocal.to_s.match(/([\+-]\d{4})/).to_s
  local_offset = local_offset_str[1..2].to_i.hr + local_offset_str[3..4].to_i.min
  local_offset_str[0..0].eql?('-') ? local_offset * -1 : local_offset
end

#lock(resource, timeout, device: nil, concurrent: 1, acquire_timeout: nil) ⇒ Object

Public: Locks the specified resource(s) to perform an action.

resource - String or Array of String resources to lock. timeout - Integer lock activity timeout in milliseconds. block - Block to perform. device - Integer device to lock (default: nil). concurrent - Integer concurrent locks allowed (default: 1). acquire_timeout - Integer lock acquisition timeout in milliseconds (default: nil).

If nil, 16 * the lock activity timeout will be used.

Returns the result of the specified block.



229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
 
# File 'lib/test_case/test_case.rb', line 229

def lock(resource, timeout, device: nil, concurrent: 1, acquire_timeout: nil)
  ret = nil
  locked_resources = []
  uri = '/api/lock'
  uri += "/#{device}" unless device.nil?
  acquire_timeout ||= (timeout * 16)
  resource = [resource] unless resource.is_a?(Array)
  begin
    resource.each do |res|
      # only lock if not already locked

      next if @locked_resources.key?(res)

      logger.debug("Attempting to lock TMC resource [#{res}] at #{timestamp}")
      resp = tmc_post(uri, json: { resource: res, timeout: timeout, concurrent: concurrent }, timeout: acquire_timeout)
      id = resp['id']
      locked_resources << id
      @locked_resources[res] = id
      add_teardown(nil, device, id) do |dev, lid|
        release_lock(dev, lid) if @locked_resources.value?(lid)
      end
    end
    begin
      Timeout.timeout(timeout.to_f / 1.sec) do
        ret = yield
      end
    rescue Timeout::Error
      raise "Lock activity timed out after #{timeout} ms!"
    end
  ensure
    locked_resources.each do |id|
      release_lock(device, id)
      logger.debug("TMC resource [#{id}] unlocked at #{timestamp}")
    end
  end
  ret
end

#name ⇒ Object

Public: Get test name.



167
168
169
170
171
172
173
 
# File 'lib/test_case/test_case.rb', line 167

def name
  if @name.nil?
    resp = tmc_get("/api/jobs/#{@job}/tests/#{@id}") || {}
    @name = test_info['name'] || self.class.name.split('::').last.gsub(/(?<!^)([A-Z]|[0-9]+)/, ' \0')
  end
  @name
end

#pass(reason = nil) ⇒ Object

Public: Passes the test or step.

reason - String message describing the reason for passing.

Returns nothing.



447
448
449
450
451
452
453
454
 
# File 'lib/test_case/test_case.rb', line 447

def pass(reason = nil)
  is_perf = is_a?(PerformanceTestCase)
  if @current_step.nil?
    @status = TestStatus.new(self, is_perf ? TestStatus::DONE : TestStatus::PASSED, reason: reason)
  else
    @current_step.pass(reason)
  end
end

#ping?(host) ⇒ Boolean

Public: Pings the given host.

host - String host name or IP to ping.

Returns true if the host could be pinged, otherwise false.

Returns:

  • (Boolean) 


340
341
342
343
 
# File 'lib/test_case/test_case.rb', line 340

def ping?(host)
  failed = orig_backticks("ping -n 1 #{host}").match(/(?i)Received\s*=\s*1/).nil?
  !failed
end

#record_error(msg, log: false, log_opts: {}, backtrace: nil) ⇒ Object

Public: Records a test error.

msg - String error message. log - Boolean indicating whether to log the error immediately (default: false). log_opts - Hash of options to pass to ‘logger.error’, if log is enabled (default: nil). backtrace - Array containing error backtrace (default: {}).

If default, caller will be used.

Returns the TestError instance.



486
487
488
489
490
491
492
493
494
 
# File 'lib/test_case/test_case.rb', line 486

def record_error(msg, log: false, log_opts: {}, backtrace: nil)
  err = TestError.new(self, msg, backtrace || caller, step: @current_step)
  @errors << err
  if log
    msg += " #{err}"
    logger.error(msg, log_opts)
  end
  err
end

#report_result(args = {}) ⇒ Object

Public: Reports a measurement to the designated system.

return_value - Integer or Float return value (default: nil).

If default, return value will be calculated from start and stop time.

start_time - Time measurement started (default: start_time). stop_time - Time measurement stopped (default: stop_time). message - String details about measurement (default: nil).

If default, a generic message will be used.

execution_metric - Execution metric for which to report (default: nil).

If default, will not report for a specific execution metric.

return_title - Return title to report (default: nil).

If default, will not report for a specific return title.

result - Symbol result to report (default: nil).

Should be :passed, :failed, :error, or :done. If default, current test status will be used.

Returns nothing.



407
408
409
410
411
412
413
414
415
416
 
# File 'lib/test_case/test_case.rb', line 407

def report_result(args = {})
  start = args.fetch(:start_time, start_time || Time.now)
  stop = args.fetch(:stop_time, stop_time || Time.now)
  args[:tcr_auto_detail_text] = args.delete(:message)
  result = args.delete(:result)
  logger.info("Reporting #{args[:message] || args[:execution_metric] || args[:return_title] || 'result'}: " \
                  "#{result.nil? ? '' : "#{result.to_s.capitalize} in "}#{args[:return_value] || (stop - start)} s")
  report_to_sword(result, start, stop, args) if is_sword?
  @reported_results = true
end

#safe_timestamp(at: nil) ⇒ Object

Public: Gets an ISO 8601-formatted timestamp safe for use in a filename.

at - DateTime or Time object (default: nil).

Returns the String timestamp.



281
282
283
284
 
# File 'lib/test_case/test_case.rb', line 281

def safe_timestamp(at: nil)
  at ||= DateTime.now
  at.strftime(TmcLogger::DATETIME_FORMAT).delete(':')
end

#server_market ⇒ Object

Public: Gets server market.

Returns the String market of the TMC server.



506
507
508
 
# File 'lib/test_case/test_case.rb', line 506

def server_market
  server_info['market']
end

#server_name ⇒ Object

Public: Gets server name.

Returns the String name of the TMC server.



499
500
501
 
# File 'lib/test_case/test_case.rb', line 499

def server_name
  server_info['name']
end

#sleep(delay) ⇒ Object

preserve the built-in sleep method so we can use it



293
294
295
296
297
298
299
300
301
 
# File 'lib/test_case/test_case.rb', line 293

def sleep(delay)
  if delay.is_a?(Range)
    min = delay.first
    range = delay.last - min
    delay = min + rand(range + 1)
  end
  logger.debug("Sleep for: [#{delay.to_i}] ms")
  builtin_sleep(delay.to_i / 1000.0)
end

#status ⇒ Object

Public: Gets test status.

Returns String status.



147
148
149
 
# File 'lib/test_case/test_case.rb', line 147

def status
  @status.status
end

#system(*_c) ⇒ Object

Public: Block probes.



162
163
164
 
# File 'lib/test_case/test_case.rb', line 162

def system(*_c)
  probe_fail('system')
end

#test_data ⇒ Object Also known as: options

Public: Gets test data.

Examples:

test_data[:foo] = 'bar'
#=> 'bar'

test_data.refresh
#=> nil

val = test_data[:foo]
#=> 'bar'

val = test_data.fetch(:foo)
#=> 'bar'

val = test_data.update(nums: {int: 1, float: 3.5})
#=> {:foo => 'bar', :nums => {:int => 1, :float => 3.5}}

val = test_data.dig(:nums, :float)
#=> 3.5

val = test_data.dig(:nums, :decimal)
#=> nil

Returns test data hash.



329
330
331
332
 
# File 'lib/test_case/test_case.rb', line 329

def test_data
  @test_data.refresh if !@test_data.promiscuous? && @test_data.empty?
  @test_data
end

#test_step(id, name: nil, &block) ⇒ Object

Public: Performs a test step.

id - Integer test step id. name - String name of the test step (default: nil). block - Block that performs the test step.

Returns nothing.



363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
 
# File 'lib/test_case/test_case.rb', line 363

def test_step(id, name: nil, &block)
  if id.nil?
    id = @steps.empty? ? 1 : @steps.last.id + 1
  end
  # Look up the step

  update = false
  step = @steps.find { |step| step.id == id && step.device == @primary_device_id }
  if step.nil?
    # Create the step if it does not exist

    step = TestStep.new(self, id, name, @primary_device_id, @iteration)
    @steps << step
    update = true
  elsif !name.nil? && name != step.name
    step.instance_variable_set(:@name, name)
    update = true
  end
  @current_step = step
  step.perform(update: update, &block)
  @current_step = nil
end

#test_steps_ignore_errors(val) ⇒ Object

Public: Sets test step errors behavior.

val - A Boolean indicating whether test steps should ignore errors.

Returns nothing.



187
188
189
190
 
# File 'lib/test_case/test_case.rb', line 187

def test_steps_ignore_errors(val)
  @test_steps_ignore_errors = val
  @steps.each {|s| s.instance_variable_set(:@raising, !val)}
end

#test_steps_ignore_errors? ⇒ Boolean

Public: Gets test step errors behavior.

Returns a Boolean indicating test steps ignore errors.

Returns:

  • (Boolean) 


178
179
180
 
# File 'lib/test_case/test_case.rb', line 178

def test_steps_ignore_errors?
  @test_steps_ignore_errors
end

#timestamp(at: nil) ⇒ Object

Public: Gets an ISO 8601-formatted timestamp.

at - DateTime or Time object (default: nil).

Returns the String timestamp.



271
272
273
274
 
# File 'lib/test_case/test_case.rb', line 271

def timestamp(at: nil)
  at ||= DateTime.now
  at.strftime(TmcLogger::DATETIME_FORMAT)
end

#to_time(obj) ⇒ Object

Public: Converts the specified object to a time.

obj - Object to convert.

Returns a Time object.



579
580
581
582
583
584
585
586
587
588
589
590
591
592
 
# File 'lib/test_case/test_case.rb', line 579

def to_time(obj)
  if obj.is_a?(Time)
    obj
  elsif obj.is_a?(DateTime)
    obj.to_time
  elsif obj.is_a?(Numeric)
    # This expects seconds since the epoch

    Time.at(obj)
  elsif obj.is_a?(String)
    obj.to_time
  else
    raise "#{obj.class.name} cannot be converted to time!"
  end
end