Class: Fluent::ForwardOutput::Node

Inherits:
Object
  • Object
show all
Defined in:
lib/fluent/plugin/out_forward.rb

Direct Known Subclasses

NoneHeartbeatNode

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(sender, server, failure:) ⇒ Node

Returns a new instance of Node.



340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
 
# File 'lib/fluent/plugin/out_forward.rb', line 340

def initialize(sender, server, failure:)
  @sender = sender
  @log = sender.log
  @compress = sender.compress

  @name = server.name
  @host = server.host
  @port = server.port
  @weight = server.weight
  @standby = server.standby
  @failure = failure
  @available = true
  @state = nil

  @usock = nil

  @username = server.username
  @password = server.password
  @shared_key = server.shared_key || (sender.security && sender.security.shared_key) || ""
  @shared_key_salt = generate_salt
  @shared_key_nonce = ""

  @unpacker = Fluent::Engine.msgpack_unpacker

  @resolved_host = nil
  @resolved_time = 0
  resolved_host  # check dns
end

Instance Attribute Details

#available ⇒ Object (readonly)

for test



373
374
375
 
# File 'lib/fluent/plugin/out_forward.rb', line 373

def available
  @available
end

#failure ⇒ Object (readonly)

for test



373
374
375
 
# File 'lib/fluent/plugin/out_forward.rb', line 373

def failure
  @failure
end

#host ⇒ Object (readonly)

Returns the value of attribute host.



371
372
373
 
# File 'lib/fluent/plugin/out_forward.rb', line 371

def host
  @host
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



371
372
373
 
# File 'lib/fluent/plugin/out_forward.rb', line 371

def name
  @name
end

#port ⇒ Object (readonly)

Returns the value of attribute port.



371
372
373
 
# File 'lib/fluent/plugin/out_forward.rb', line 371

def port
  @port
end

#sockaddr ⇒ Object (readonly)

used by on_heartbeat



372
373
374
 
# File 'lib/fluent/plugin/out_forward.rb', line 372

def sockaddr
  @sockaddr
end

#standby ⇒ Object (readonly)

Returns the value of attribute standby.



371
372
373
 
# File 'lib/fluent/plugin/out_forward.rb', line 371

def standby
  @standby
end

#state ⇒ Object (readonly)

Returns the value of attribute state.



371
372
373
 
# File 'lib/fluent/plugin/out_forward.rb', line 371

def state
  @state
end

#usock ⇒ Object

Returns the value of attribute usock.



369
370
371
 
# File 'lib/fluent/plugin/out_forward.rb', line 369

def usock
  @usock
end

#weight ⇒ Object (readonly)

Returns the value of attribute weight.



371
372
373
 
# File 'lib/fluent/plugin/out_forward.rb', line 371

def weight
  @weight
end

Instance Method Details

#available? ⇒ Boolean

Returns:

  • (Boolean) 


375
376
377
 
# File 'lib/fluent/plugin/out_forward.rb', line 375

def available?
  @available
end

#check_helo(message) ⇒ Object 



613
614
615
616
617
618
619
620
621
622
623
624
 
# File 'lib/fluent/plugin/out_forward.rb', line 613

def check_helo(message)
  @log.debug "checking helo"
  # ['HELO', options(hash)]
  unless message.size == 2 && message[0] == 'HELO'
    return false
  end
  opts = message[1] || {}
  # make shared_key_check failed (instead of error) if protocol version mismatch exist
  @shared_key_nonce = opts['nonce'] || ''
  @authentication = opts['auth'] || ''
  true
end

#check_pong(message) ⇒ Object 



645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
 
# File 'lib/fluent/plugin/out_forward.rb', line 645

def check_pong(message)
  @log.debug "checking pong"
  # ['PONG', bool(authentication result), 'reason if authentication failed',
  #  self_hostname, sha512\_hex(salt + self_hostname + nonce + sharedkey)]
  unless message.size == 5 && message[0] == 'PONG'
    return false, 'invalid format for PONG message'
  end
  _pong, auth_result, reason, hostname, shared_key_hexdigest = message

  unless auth_result
    return false, 'authentication failed: ' + reason
  end

  if hostname == @sender.security.self_hostname
    return false, 'same hostname between input and output: invalid configuration'
  end

  clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(@shared_key_nonce).update(@shared_key).hexdigest
  unless shared_key_hexdigest == clientside
    return false, 'shared key mismatch'
  end

  return true, nil
end

#connect ⇒ Object 



387
388
389
 
# File 'lib/fluent/plugin/out_forward.rb', line 387

def connect
  TCPSocket.new(resolved_host, port)
end

#disable! ⇒ Object 



379
380
381
 
# File 'lib/fluent/plugin/out_forward.rb', line 379

def disable!
  @available = false
end

#establish_connection(sock) ⇒ Object 



401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
 
# File 'lib/fluent/plugin/out_forward.rb', line 401

def establish_connection(sock)
  while available? && @state != :established
    begin
      # TODO: On Ruby 2.2 or earlier, read_nonblock doesn't work expectedly.
      # We need rewrite around here using new socket/server plugin helper.
      buf = sock.read_nonblock(@sender.read_length)
      if buf.empty?
        sleep @sender.read_interval
        next
      end
      @unpacker.feed_each(buf) do |data|
        on_read(sock, data)
      end
    rescue IO::WaitReadable
      # If the exception is Errno::EWOULDBLOCK or Errno::EAGAIN, it is extended by IO::WaitReadable.
      # So IO::WaitReadable can be used to rescue the exceptions for retrying read_nonblock.
      # http://docs.ruby-lang.org/en/2.3.0/IO.html#method-i-read_nonblock
      sleep @sender.read_interval unless @state == :established
    rescue SystemCallError => e
      @log.warn "disconnected by error", host: @host, port: @port, error: e
      disable!
      break
    rescue EOFError
      @log.warn "disconnected", host: @host, port: @port
      disable!
      break
    end
  end
end

#generate_ping ⇒ Object 



626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
 
# File 'lib/fluent/plugin/out_forward.rb', line 626

def generate_ping
  @log.debug "generating ping"
  # ['PING', self_hostname, sharedkey\_salt, sha512\_hex(sharedkey\_salt + self_hostname + nonce + shared_key),
  #  username || '', sha512\_hex(auth\_salt + username + password) || '']
  shared_key_hexdigest = Digest::SHA512.new.update(@shared_key_salt)
    .update(@sender.security.self_hostname)
    .update(@shared_key_nonce)
    .update(@shared_key)
    .hexdigest
  ping = ['PING', @sender.security.self_hostname, @shared_key_salt, shared_key_hexdigest]
  if !@authentication.empty?
    password_hexdigest = Digest::SHA512.new.update(@authentication).update(@username).update(@password).hexdigest
    ping.push(@username, password_hexdigest)
  else
    ping.push('','')
  end
  ping
end

#generate_salt ⇒ Object 



609
610
611
 
# File 'lib/fluent/plugin/out_forward.rb', line 609

def generate_salt
  SecureRandom.hex(16)
end

#heartbeat(detect = true) ⇒ Object 



592
593
594
595
596
597
598
599
600
601
602
 
# File 'lib/fluent/plugin/out_forward.rb', line 592

def heartbeat(detect=true)
  now = Time.now.to_f
  @failure.add(now)
  if detect && !@available && @failure.sample_size > @sender.recover_sample_size
    @available = true
    @log.warn "recovered forwarding server '#{@name}'", host: @host, port: @port
    true
  else
    nil
  end
end

#on_read(sock, data) ⇒ Object 



670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
 
# File 'lib/fluent/plugin/out_forward.rb', line 670

def on_read(sock, data)
  @log.trace __callee__

  case @state
  when :helo
    unless check_helo(data)
      @log.warn "received invalid helo message from #{@name}"
      disable! # shutdown
      return
    end
    sock.write(generate_ping.to_msgpack)
    @state = :pingpong
  when :pingpong
    succeeded, reason = check_pong(data)
    unless succeeded
      @log.warn "connection refused to #{@name}: #{reason}"
      disable! # shutdown
      return
    end
    @state = :established
    @log.debug "connection established", host: @host, port: @port
  else
    raise "BUG: unknown session state: #{@state}"
  end
end

#resolved_host ⇒ Object 



533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
 
# File 'lib/fluent/plugin/out_forward.rb', line 533

def resolved_host
  case @sender.expire_dns_cache
  when 0
    # cache is disabled
    resolve_dns!

  when nil
    # persistent cache
    @resolved_host ||= resolve_dns!

  else
    now = Engine.now
    rh = @resolved_host
    if !rh || now - @resolved_time >= @sender.expire_dns_cache
      rh = @resolved_host = resolve_dns!
      @resolved_time = now
    end
    rh
  end
end

#send_data(tag, chunk) ⇒ Object 



431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
 
# File 'lib/fluent/plugin/out_forward.rb', line 431

def send_data(tag, chunk)
  sock = connect
  @state = @sender.security ? :helo : :established
  begin
    set_socket_options(sock)

    if @state != :established
      establish_connection(sock)
    end

    unless available?
      raise ForwardOutputConnectionClosedError, "failed to establish connection with node #{@name}"
    end

    option = { 'size' => chunk.size_of_events, 'compressed' => @compress }
    option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response

    # out_forward always uses Raw32 type for content.
    # Raw16 can store only 64kbytes, and it should be much smaller than buffer chunk size.

    sock.write @sender.forward_header        # beginArray(3)
    sock.write tag.to_msgpack                # 1. writeRaw(tag)
    chunk.open(compressed: @compress) do |chunk_io|
      sock.write [0xdb, chunk_io.size].pack('CN') # 2. beginRaw(size) raw32
      IO.copy_stream(chunk_io, sock)              # writeRawBody(packed_es)
    end
    sock.write option.to_msgpack             # 3. writeOption(option)

    if @sender.require_ack_response
      # Waiting for a response here results in a decrease of throughput because a chunk queue is locked.
      # To avoid a decrease of throughput, it is necessary to prepare a list of chunks that wait for responses
      # and process them asynchronously.
      if IO.select([sock], nil, nil, @sender.ack_response_timeout)
        raw_data = begin
                     sock.recv(1024)
                   rescue Errno::ECONNRESET
                     ""
                   end

        # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
        # If this happens we assume the data wasn't delivered and retry it.
        if raw_data.empty?
          @log.warn "node closed the connection. regard it as unavailable.", host: @host, port: @port
          disable!
          raise ForwardOutputConnectionClosedError, "node #{@host}:#{@port} closed connection"
        else
          @unpacker.feed(raw_data)
          res = @unpacker.read
          if res['ack'] != option['chunk']
            # Some errors may have occured when ack and chunk id is different, so send the chunk again.
            raise ForwardOutputResponseError, "ack in response and chunk id in sent data are different"
          end
        end

      else
        # IO.select returns nil on timeout.
        # There are 2 types of cases when no response has been received:
        # (1) the node does not support sending responses
        # (2) the node does support sending response but responses have not arrived for some reasons.
        @log.warn "no response from node. regard it as unavailable.", host: @host, port: @port
        disable!
        raise ForwardOutputACKTimeoutError, "node #{host}:#{port} does not return ACK"
      end
    end

    heartbeat(false)
    res  # for test
  ensure
    sock.close_write
    sock.close
  end
end

#send_heartbeat ⇒ Object

FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ”.to_msgpack + [].to_msgpack



505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
 
# File 'lib/fluent/plugin/out_forward.rb', line 505

def send_heartbeat
  case @sender.heartbeat_type
  when :tcp
    sock = connect
    begin
      opt = [1, @sender.send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
      sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
      # opt = [@sender.send_timeout.to_i, 0].pack('L!L!')  # struct timeval
      # sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)

      ## don't send any data to not cause a compatibility problem
      # sock.write FORWARD_TCP_HEARTBEAT_DATA

      # successful tcp connection establishment is considered as valid heartbeat
      heartbeat(true)
    ensure
      sock.close_write
      sock.close
    end
  when :udp
    @usock.send "\0", 0, Socket.pack_sockaddr_in(@port, resolved_host)
  when :none # :none doesn't use this class
    raise "BUG: heartbeat_type none must not use Node"
  else
    raise "BUG: unknown heartbeat_type '#{@sender.heartbeat_type}'"
  end
end

#set_socket_options(sock) ⇒ Object 



391
392
393
394
395
396
397
398
399
 
# File 'lib/fluent/plugin/out_forward.rb', line 391

def set_socket_options(sock)
  opt = [1, @sender.send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
  sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)

  opt = [@sender.send_timeout.to_i, 0].pack('L!L!')  # struct timeval
  sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)

  sock
end

#standby? ⇒ Boolean

Returns:

  • (Boolean) 


383
384
385
 
# File 'lib/fluent/plugin/out_forward.rb', line 383

def standby?
  @standby
end

#tick ⇒ Object 



562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
 
# File 'lib/fluent/plugin/out_forward.rb', line 562

def tick
  now = Time.now.to_f
  if !@available
    if @failure.hard_timeout?(now)
      @failure.clear
    end
    return nil
  end

  if @failure.hard_timeout?(now)
    @log.warn "detached forwarding server '#{@name}'", host: @host, port: @port, hard_timeout: true
    @available = false
    @resolved_host = nil  # expire cached host
    @failure.clear
    return true
  end

  if @sender.phi_failure_detector
    phi = @failure.phi(now)
    if phi > @sender.phi_threshold
      @log.warn "detached forwarding server '#{@name}'", host: @host, port: @port, phi: phi, phi_threshold: @sender.phi_threshold
      @available = false
      @resolved_host = nil  # expire cached host
      @failure.clear
      return true
    end
  end
  false
end

#to_msgpack(out = '') ⇒ Object

TODO: #to_msgpack(string) is deprecated



605
606
607
 
# File 'lib/fluent/plugin/out_forward.rb', line 605

def to_msgpack(out = '')
  [@host, @port, @weight, @available].to_msgpack(out)
end