Class: PseudoCleaner::RedisMonitorCleaner

Inherits:
Object
  • Object
show all
Defined in:
lib/pseudo_cleaner/redis_monitor_cleaner.rb

Overview

I’m not a huge fan of sleeps. In the non-rails world, I used to be able to do a sleep(0) to signal the system to check if somebody else needed to do some work. Testing with Rails, I find I have to actually sleep, so I do a very short time like 0.01.

Defined Under Namespace

Classes: RedisMessage

Constant Summary collapse

FLUSH_COMMANDS =

SUITE_KEY = “PseudoDelete::RedisMonitorCleaner:initial_redis_state”

[
    "flushall",
    "flushdb"
]
WRITE_COMMANDS = 
[
    "append",
    "bitop",
    "blpop",
    "brpop",
    "brpoplpush",
    "decr",
    "decrby",
    "del",
    "expire",
    "expireat",
    "getset",
    "hset",
    "hsetnx",
    "hincrby",
    "hincrbyfloat",
    "hmset",
    "hdel",
    "incr",
    "incrby",
    "incrbyfloat",
    "linsert",
    "lpop",
    "lpush",
    "lpushx",
    "lrem",
    "lset",
    "ltrim",
    "mapped_hmset",
    "mapped_mset",
    "mapped_msetnx",
    "move",
    "mset",
    "msetnx",
    "persist",
    "pexpire",
    "pexpireat",
    "psetex",
    "rename",
    "renamenx",
    "restore",
    "rpop",
    "rpoplpush",
    "rpush",
    "rpushx",
    "sadd",
    "sdiffstore",
    "set",
    "setbit",
    "setex",
    "setnx",
    "setrange",
    "sinterstore",
    "smove",
    "sort",
    "spop",
    "srem",
    "sunionstore",
    "zadd",
    "zincrby",
    "zinterstore",
    "zrem",
    "zremrangebyrank",
    "zremrangebyscore",
]
READ_COMMANDS = 
[
    "bitcount",
    "bitop",
    "dump",
    "exists",
    "get",
    "getbit",
    "getrange",
    "hget",
    "hmget",
    "hexists",
    "hlen",
    "hkeys",
    "hscan",
    "hscan_each",
    "hvals",
    "hgetall",
    "lindex",
    "llen",
    "lrange",
    "mapped_hmget",
    "mapped_mget",
    "mget",
    "persist",
    "scard",
    "scan",
    "scan_each",
    "sdiff",
    "sismember",
    "smembers",
    "srandmember",
    "sscan",
    "sscan_each",
    "strlen",
    "sunion",
    "type",
    "zcard",
    "zcount",
    "zrange",
    "zrangebyscore",
    "zrank",
    "zrevrange",
    "zrevrangebyscore",
    "zrevrank",
    "zscan",
    "zscan_each",
    "zscore",
]

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(start_method, end_method, table, options) ⇒ RedisMonitorCleaner

Returns a new instance of RedisMonitorCleaner.



315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 315

def initialize(start_method, end_method, table, options)
  @initial_keys       = SortedSet.new
  @monitor_thread     = nil
  @redis_name         = nil
  @suite_altered_keys = SortedSet.new

  unless PseudoCleaner::MasterCleaner::VALID_START_METHODS.include?(start_method)
    raise "You must specify a valid start function from: #{PseudoCleaner::MasterCleaner::VALID_START_METHODS}."
  end
  unless PseudoCleaner::MasterCleaner::VALID_END_METHODS.include?(end_method)
    raise "You must specify a valid end function from: #{PseudoCleaner::MasterCleaner::VALID_END_METHODS}."
  end

  @options = options

  @options[:table_start_method] ||= start_method
  @options[:table_end_method]   ||= end_method
  @options[:output_diagnostics] ||= PseudoCleaner::Configuration.current_instance.output_diagnostics ||
      PseudoCleaner::Configuration.current_instance.post_transaction_analysis

  @redis = table
end

Instance Attribute Details

#initial_keys ⇒ Object (readonly)

Returns the value of attribute initial_keys.



164
165
166
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 164

def initial_keys
  @initial_keys
end

#monitor_thread ⇒ Object (readonly)

Returns the value of attribute monitor_thread.



163
164
165
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 163

def monitor_thread
  @monitor_thread
end

#options ⇒ Object

Returns the value of attribute options.



165
166
167
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 165

def options
  @options
end

Instance Method Details

#<=>(right_object) ⇒ Object 



338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 338

def <=>(right_object)
  if (right_object.is_a?(PseudoCleaner::RedisMonitorCleaner))
    return 0
  elsif (right_object.is_a?(PseudoCleaner::TableCleaner))
    return 1
  else
    if right_object.respond_to?(:<=>)
      comparison = (right_object <=> self)
      if comparison
        return -1 * comparison
      end
    end
  end

  return 1
end

#ignore_key(key) ⇒ Object 



431
432
433
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 431

def ignore_key(key)
  ignore_regexes.detect { |ignore_regex| key =~ ignore_regex }
end

#ignore_regexes ⇒ Object 



427
428
429
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 427

def ignore_regexes
  []
end

#peek_values ⇒ Object 



456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 456

def peek_values
  synchronize_test_values do |updated_values|
    if updated_values && !updated_values.empty?
      output_values = false

      if PseudoCleaner::MasterCleaner.report_table
        Cornucopia::Util::ReportTable.new(nested_table:         PseudoCleaner::MasterCleaner.report_table,
                                          nested_table_label:   redis_name,
                                          suppress_blank_table: true) do |report_table|
          updated_values.each do |updated_value|
            unless ignore_key(updated_value)
              output_values = true
              report_table.write_stats updated_value, report_record(updated_value)
            end
          end
        end
      else
        PseudoCleaner::Logger.write("  #{redis_name}")

        updated_values.each do |updated_value|
          unless ignore_key(updated_value)
            output_values = true
            PseudoCleaner::Logger.write("    #{updated_value}: #{report_record(updated_value)}")
          end
        end
      end

      PseudoCleaner::MasterCleaner.report_error if output_values
    end
  end
end

#queue ⇒ Object 



529
530
531
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 529

def queue
  @queue ||= Queue.new
end

#redis ⇒ Object 



355
356
357
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 355

def redis
  @redis ||= Redis.current
end

#redis_name ⇒ Object 



435
436
437
438
439
440
441
442
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 435

def redis_name
  unless @redis_name
    redis_options = redis.client.options.with_indifferent_access
    @redis_name   = "#{redis_options[:host]}:#{redis_options[:port]}/#{redis_options[:db]}"
  end

  @redis_name
end

#report_dirty_values(message, test_values) ⇒ Object 



635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 635

def report_dirty_values message, test_values
  if test_values && !test_values.empty?
    output_values = false

    if PseudoCleaner::MasterCleaner.report_table
      Cornucopia::Util::ReportTable.new(nested_table:         PseudoCleaner::MasterCleaner.report_table,
                                        nested_table_label:   redis_name,
                                        suppress_blank_table: true) do |report_table|
        report_table.write_stats "action", message
        test_values.each_with_index do |key_name, index|
          unless ignore_key(key_name)
            output_values = true
            report_table.write_stats index, report_record(key_name)
          end
        end
      end
    else
      PseudoCleaner::Logger.write("********* RedisMonitorCleaner - #{message}".red.on_light_white)
      test_values.each do |key_name|
        unless ignore_key(key_name)
          output_values = true
          PseudoCleaner::Logger.write("  #{key_name}: #{report_record(key_name)}".red.on_light_white)
        end
      end
    end

    PseudoCleaner::MasterCleaner.report_error if output_values
  end
end

#report_end_of_suite_state(report_reason) ⇒ Object 



496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 496

def report_end_of_suite_state report_reason
  current_keys = SortedSet.new(redis.keys)

  deleted_keys = initial_keys - current_keys
  new_keys     = current_keys - initial_keys

  # filter out values we inserted that will go away on their own.
  new_keys     = new_keys.select { |key| (key =~ /redis_cleaner::synchronization_(?:end_)?key_[0-9]+_[0-9]+/).nil? }

  report_dirty_values "new values as of #{report_reason}", new_keys
  report_dirty_values "values deleted before #{report_reason}", deleted_keys
  report_dirty_values "initial values changed during suite run", @suite_altered_keys

  @suite_altered_keys = SortedSet.new

  new_keys.each do |key_value|
    redis.del key_value
  end
end

#report_record(key_name) ⇒ Object 



609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 609

def report_record(key_name)
  key_hash = { key: key_name, type: redis.type(key_name), ttl: redis.ttl(key_name) }
  case key_hash[:type]
    when "string"
      key_hash[:value] = redis.get(key_name)
    when "list"
      key_hash[:list] = { len: redis.llen(key_name), values: redis.lrange(key_name, 0, -1) }
    when "set"
      key_hash[:set] = redis.smembers(key_name)
    when "zset"
      key_hash[:sorted_set] = redis.smembers(key_name)
    when "hash"
      key_hash[:list] = { len: redis.hlen(key_name), values: redis.hgetall(key_name) }
  end

  if key_hash[:value].nil? &&
      key_hash[:list].nil? &&
      key_hash[:set].nil? &&
      key_hash[:sorted_set].nil? &&
      key_hash[:hash].nil?
    key_hash[:value] = "[[DELETED]]"
  end

  key_hash
end

#reset_suite ⇒ Object 



417
418
419
420
421
422
423
424
425
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 417

def reset_suite
  report_end_of_suite_state "reset suite"

  if monitor_thread
    monitor_thread.kill
    @monitor_thread = nil
    start_monitor
  end
end

#review_rows(&block) ⇒ Object 



444
445
446
447
448
449
450
451
452
453
454
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 444

def review_rows(&block)
  synchronize_test_values do |updated_values|
    if updated_values && !updated_values.empty?
      updated_values.each do |updated_value|
        unless ignore_key(updated_value)
          block.yield redis_name, report_record(updated_value)
        end
      end
    end
  end
end

#start_monitor ⇒ Object 



533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 533

def start_monitor
  cleaner_class = self

  @initial_keys = SortedSet.new(redis.keys)
  # @initial_keys.add(PseudoCleaner::RedisMonitorCleaner::SUITE_KEY)
  # @initial_keys.each do |key_value|
  #   redis.sadd(PseudoCleaner::RedisMonitorCleaner::SUITE_KEY, key_value)
  # end
  if @options[:output_diagnostics]
    if PseudoCleaner::MasterCleaner.report_table
      Cornucopia::Util::ReportTable.new(nested_table:         PseudoCleaner::MasterCleaner.report_table,
                                        nested_table_label:   redis_name,
                                        suppress_blank_table: true) do |report_table|
        report_table.write_stats "initial keys count", @initial_keys.count
      end
    else
      PseudoCleaner::Logger.write("#{redis_name}")
      PseudoCleaner::Logger.write("    Initial keys count - #{@initial_keys.count}")
    end
  end

  unless @monitor_thread
    @monitor_thread = Thread.new do
      in_redis_cleanup = false
      updated_keys     = SortedSet.new

      monitor_redis    = Redis.new(cleaner_class.redis.client.options)
      redis_options    = monitor_redis.client.options.with_indifferent_access
      cleaner_class_db = redis_options[:db]

      monitor_redis.monitor do |message|
        redis_message = RedisMessage.new message

        if redis_message.db == cleaner_class_db
          process_command = true

          if redis_message.command == "setex"
            if redis_message.keys[0] == cleaner_class.synchronize_key
              process_command = false

              in_redis_cleanup = true
              return_values    = updated_keys
              updated_keys     = SortedSet.new
              cleaner_class.queue << return_values
            elsif redis_message.keys[0] == cleaner_class.synchronize_end_key
              in_redis_cleanup                       = false
              cleaner_class.monitor_thread[:updated] = nil
              process_command                        = false
            end
          elsif redis_message.command == "del"
            if in_redis_cleanup
              process_command = false
            end
          end

          if process_command
            # flush...
            if PseudoCleaner::RedisMonitorCleaner::WRITE_COMMANDS.include? redis_message.command
              updated_keys.merge(redis_message.keys)
            elsif PseudoCleaner::RedisMonitorCleaner::FLUSH_COMMANDS.include? redis_message.command
              # Not sure I can get the keys at this point...
              # updated_keys.merge(cleaner_class.redis.keys)
            end
          end
        elsif "flushall" == redis_message.command
          # Not sure I can get the keys at this point...
          # updated_keys.merge(cleaner_class.redis.keys)
        end
      end
    end

    sleep(0.01)
    redis.get(synchronize_key)
  end
end

#suite_end(test_strategy) ⇒ Object 



408
409
410
411
412
413
414
415
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 408

def suite_end test_strategy
  report_end_of_suite_state "suite end"

  if monitor_thread
    monitor_thread.kill
    @monitor_thread = nil
  end
end

#suite_start(test_strategy) ⇒ Object 



359
360
361
362
363
364
365
366
367
368
369
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 359

def suite_start test_strategy
  @test_strategy ||= test_strategy

  # if redis.type(PseudoCleaner::RedisMonitorCleaner::SUITE_KEY) == "set"
  #   @initial_keys = SortedSet.new(redis.smembers(PseudoCleaner::RedisMonitorCleaner::SUITE_KEY))
  #   report_end_of_suite_state "before suite start"
  # end
  # redis.del PseudoCleaner::RedisMonitorCleaner::SUITE_KEY

  start_monitor
end

#synchronize_end_key ⇒ Object 



492
493
494
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 492

def synchronize_end_key
  @synchronize_end_key ||= "redis_cleaner::synchronization_end_key_#{rand(1..1_000_000_000_000_000_000)}_#{rand(1..1_000_000_000_000_000_000)}"
end

#synchronize_key ⇒ Object 



488
489
490
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 488

def synchronize_key
  @synchronize_key ||= "redis_cleaner::synchronization_key_#{rand(1..1_000_000_000_000_000_000)}_#{rand(1..1_000_000_000_000_000_000)}"
end

#synchronize_test_values(&block) ⇒ Object 



516
517
518
519
520
521
522
523
524
525
526
527
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 516

def synchronize_test_values(&block)
  updated_values = nil

  if monitor_thread
    redis.setex(synchronize_key, 1, true)
    updated_values = queue.pop
  end

  block.yield updated_values

  redis.setex(synchronize_end_key, 1, true)
end

#test_end(test_strategy) ⇒ Object 



385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 385

def test_end test_strategy
  synchronize_test_values do |updated_values|
    if updated_values && !updated_values.empty?
      report_keys = []

      if @options[:output_diagnostics]
        report_dirty_values "updated values", updated_values
      end

      updated_values.each do |value|
        if initial_keys.include?(value)
          report_keys << value
          @suite_altered_keys << value
        else
          redis.del(value)
        end
      end

      report_dirty_values "initial values altered by test", report_keys
    end
  end
end

#test_start(test_strategy) ⇒ Object 



371
372
373
374
375
376
377
378
379
380
381
382
383
 
# File 'lib/pseudo_cleaner/redis_monitor_cleaner.rb', line 371

def test_start test_strategy
  @test_strategy ||= test_strategy

  synchronize_test_values do |test_values|
    if test_values && !test_values.empty?
      report_dirty_values "values altered before the test started", test_values

      test_values.each do |value|
        redis.del value unless initial_keys.include?(value)
      end
    end
  end
end