Class: Gitlab::Database::LoadBalancing::ServiceDiscovery

Inherits:
Object
  • Object
show all
Defined in:
lib/gitlab/database/load_balancing/service_discovery.rb,
lib/gitlab/database/load_balancing/service_discovery/sampler.rb

Overview

Service discovery of secondary database hosts.

Service discovery works by periodically looking up a DNS record. If the DNS record returns a new list of hosts, this class will update the load balancer with said hosts. Requests may continue to use the old hosts until they complete.

Defined Under Namespace

Classes: Address, Sampler

Constant Summary collapse

EmptyDnsResponse = 
Class.new(StandardError)
MAX_SLEEP_ADJUSTMENT = 
10
MAX_DISCOVERY_RETRIES = 
3
DISCOVERY_THREAD_REFRESH_DELTA = 
5
RETRY_DELAY_RANGE = 
(0.1..0.2)
RECORD_TYPES = 
{
  'A' => Net::DNS::A,
  'SRV' => Net::DNS::SRV
}.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(load_balancer, nameserver:, port:, record:, record_type: 'A', interval: 60, disconnect_timeout: 120, use_tcp: false, max_replica_pools: nil) ⇒ ServiceDiscovery

nameserver - The nameserver to use for DNS lookups. port - The port of the nameserver. record - The DNS record to look up for retrieving the secondaries. record_type - The type of DNS record to look up interval - The time to wait between lookups. disconnect_timeout - The time after which an old host should be

forcefully disconnected.

use_tcp - Use TCP instaed of UDP to look up resources load_balancer - The load balancer instance to use rubocop:disable Metrics/ParameterLists



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 54

def initialize(
  load_balancer,
  nameserver:,
  port:,
  record:,
  record_type: 'A',
  interval: 60,
  disconnect_timeout: 120,
  use_tcp: false,
  max_replica_pools: nil
)
  @nameserver = nameserver
  @port = port
  @record = record
  @record_type = record_type_for(record_type)
  @interval = interval
  @disconnect_timeout = disconnect_timeout
  @use_tcp = use_tcp
  @load_balancer = load_balancer
  @max_replica_pools = max_replica_pools
  @nameserver_ttl = 1.second.ago # Begin with an expired ttl to trigger a nameserver dns lookup
end

Instance Attribute Details

#disconnect_timeout ⇒ Object (readonly)

Returns the value of attribute disconnect_timeout.



20
21
22
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 20

def disconnect_timeout
  @disconnect_timeout
end

#interval ⇒ Object (readonly)

Returns the value of attribute interval.



20
21
22
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 20

def interval
  @interval
end

#load_balancer ⇒ Object (readonly)

Returns the value of attribute load_balancer.



20
21
22
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 20

def load_balancer
  @load_balancer
end

#record ⇒ Object (readonly)

Returns the value of attribute record.



20
21
22
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 20

def record
  @record
end

#record_type ⇒ Object (readonly)

Returns the value of attribute record_type.



20
21
22
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 20

def record_type
  @record_type
end

#refresh_thread ⇒ Object

Returns the value of attribute refresh_thread.



18
19
20
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 18

def refresh_thread
  @refresh_thread
end

#refresh_thread_interruption_logged ⇒ Object

Returns the value of attribute refresh_thread_interruption_logged.



18
19
20
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 18

def refresh_thread_interruption_logged
  @refresh_thread_interruption_logged
end

#refresh_thread_last_run ⇒ Object

Returns the value of attribute refresh_thread_last_run.



18
19
20
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 18

def refresh_thread_last_run
  @refresh_thread_last_run
end

Instance Method Details

#addresses_from_dns ⇒ Object

Returns an Array containing:

  1. The time to wait for the next check.

  2. An array containing the hostnames of the DNS record.

Raises:



161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 161

def addresses_from_dns
  response = resolver.search(record, record_type)
  resources = response.answer

  addresses =
    case record_type
    when Net::DNS::A
      addresses_from_a_record(resources)
    when Net::DNS::SRV
      addresses_from_srv_record(response)
    end

  addresses = sampler.sample(addresses)

  raise EmptyDnsResponse if addresses.empty?

  # Addresses are sorted so we can directly compare the old and new
  # addresses, without having to use any additional data structures.
  [new_wait_time_for(resources), addresses.sort]
end

#addresses_from_load_balancer ⇒ Object 



190
191
192
193
194
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 190

def addresses_from_load_balancer
  load_balancer.host_list.host_names_and_ports.map do |hostname, port|
    Address.new(hostname, port)
  end.sort
end

#log_refresh_thread_interruption ⇒ Object 



210
211
212
213
214
215
216
217
218
219
220
221
222
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 210

def log_refresh_thread_interruption
  return if refresh_thread_last_run.blank? || refresh_thread_interruption_logged ||
    (refresh_thread_last_run + DISCOVERY_THREAD_REFRESH_DELTA.minutes).future?

  Gitlab::Database::LoadBalancing::Logger.error(
    event: :service_discovery_refresh_thread_interrupt,
    refresh_thread_last_run: refresh_thread_last_run,
    thread_status: refresh_thread&.status&.to_s,
    thread_backtrace: refresh_thread&.backtrace&.join('\n')
  )

  self.refresh_thread_interruption_logged = true
end

#new_wait_time_for(resources) ⇒ Object 



182
183
184
185
186
187
188
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 182

def new_wait_time_for(resources)
  wait = resources.first&.ttl || interval

  # The preconfigured interval acts as a minimum amount of time to
  # wait.
  wait < interval ? interval : wait
end

#perform_service_discovery ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 93

def perform_service_discovery
  MAX_DISCOVERY_RETRIES.times do
    return refresh_if_necessary
  rescue StandardError => error
    # Any exceptions that might occur should be reported to
    # Sentry, instead of silently terminating this thread.
    Gitlab::ErrorTracking.track_exception(error)

    Gitlab::Database::LoadBalancing::Logger.error(
      event: :service_discovery_failure,
      message: "Service discovery encountered an error: #{error.message}",
      host_list_length: load_balancer.host_list.length
    )

    # Slightly randomize the retry delay so that, in the case of a total
    # dns outage, all starting services do not pressure the dns server at the same time.
    sleep(rand(RETRY_DELAY_RANGE))
  end

  interval
end

#refresh_if_necessary ⇒ Object

Refreshes the hosts, but only if the DNS record returned a new list of addresses.

The return value is the amount of time (in seconds) to wait before checking the DNS record for any changes.



120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 120

def refresh_if_necessary
  wait_time, from_dns = addresses_from_dns

  current = addresses_from_load_balancer

  if from_dns != current
    ::Gitlab::Database::LoadBalancing::Logger.info(
      event: :host_list_update,
      message: "Updating the host list for service discovery",
      host_list_length: from_dns.length,
      old_host_list_length: current.length
    )
    replace_hosts(from_dns)
  end

  wait_time
end

#replace_hosts(addresses) ⇒ Object

Replaces all the hosts in the load balancer with the new ones, disconnecting the old connections.

addresses - An Array of Address structs to use for the new hosts.



142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 142

def replace_hosts(addresses)
  old_hosts = load_balancer.host_list.hosts

  load_balancer.host_list.hosts = addresses.map do |addr|
    Host.new(addr.hostname, load_balancer, port: addr.port)
  end

  # We must explicitly disconnect the old connections, otherwise we may
  # leak database connections over time. For example, if a request
  # started just before we added the new hosts it will use an old
  # host/connection. While this connection will be checked in and out,
  # it won't be explicitly disconnected.
  disconnect_old_hosts(old_hosts)
end

#resolver ⇒ Object 



196
197
198
199
200
201
202
203
204
205
206
207
208
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 196

def resolver
  return @resolver if defined?(@resolver) && @nameserver_ttl.future?

  response = Resolver.new(@nameserver).resolve

  @nameserver_ttl = response.ttl

  @resolver = Net::DNS::Resolver.new(
    nameservers: response.address,
    port: @port,
    use_tcp: @use_tcp
  )
end

#start ⇒ Object

rubocop:enable Metrics/ParameterLists



78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/gitlab/database/load_balancing/service_discovery.rb', line 78

def start
  self.refresh_thread = Thread.new do
    loop do
      self.refresh_thread_last_run = Time.current

      next_sleep_duration = perform_service_discovery

      # We slightly randomize the sleep() interval. This should reduce
      # the likelihood of _all_ processes refreshing at the same time,
      # possibly putting unnecessary pressure on the DNS server.
      sleep(next_sleep_duration + rand(MAX_SLEEP_ADJUSTMENT))
    end
  end
end