Class: Evinrude

Inherits:
Object
  • Object
show all
Includes:
LoggingHelpers
Defined in:
lib/evinrude.rb,
lib/evinrude/log.rb,
lib/evinrude/peer.rb,
lib/evinrude/backoff.rb,
lib/evinrude/message.rb,
lib/evinrude/metrics.rb,
lib/evinrude/network.rb,
lib/evinrude/resolver.rb,
lib/evinrude/snapshot.rb,
lib/evinrude/log_entry.rb,
lib/evinrude/node_info.rb,
lib/evinrude/state_machine.rb,
lib/evinrude/log_entry/null.rb,
lib/evinrude/logging_helpers.rb,
lib/evinrude/network/protocol.rb,
lib/evinrude/message/join_reply.rb,
lib/evinrude/message/read_reply.rb,
lib/evinrude/message/vote_reply.rb,
lib/evinrude/network/connection.rb,
lib/evinrude/message/join_request.rb,
lib/evinrude/message/read_request.rb,
lib/evinrude/message/vote_request.rb,
lib/evinrude/cluster_configuration.rb,
lib/evinrude/message/command_reply.rb,
lib/evinrude/state_machine/register.rb,
lib/evinrude/message/command_request.rb,
lib/evinrude/config_change_queue_entry.rb,
lib/evinrude/message/node_removal_reply.rb,
lib/evinrude/message/append_entries_reply.rb,
lib/evinrude/message/node_removal_request.rb,
lib/evinrude/message/append_entries_request.rb,
lib/evinrude/message/install_snapshot_reply.rb,
lib/evinrude/log_entry/cluster_configuration.rb,
lib/evinrude/log_entry/state_machine_command.rb,
lib/evinrude/message/install_snapshot_request.rb,
lib/evinrude/config_change_queue_entry/add_node.rb,
lib/evinrude/config_change_queue_entry/remove_node.rb

Defined Under Namespace

Modules: LoggingHelpers Classes: Backoff, ClusterConfiguration, ConfigChangeQueueEntry, Error, Log, LogEntry, Message, Metrics, Network, NoLeaderError, NodeExpiredError, NodeInfo, Peer, Resolver, Snapshot, StateMachine

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(join_hints: [], shared_keys:, state_machine: Evinrude::StateMachine::Register, logger: Logger.new("/dev/null"), node_name: nil, storage_dir: nil, heartbeat_interval: 0.25, heartbeat_timeout: 1..2, listen: {}, advertise: {}, metrics_registry: Prometheus::Client::Registry.new) ⇒ Evinrude

Returns a new instance of Evinrude.



22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/evinrude.rb', line 22

def initialize(join_hints: [], shared_keys:, state_machine: Evinrude::StateMachine::Register,
               logger: Logger.new("/dev/null"), node_name: nil, storage_dir: nil,
               heartbeat_interval: 0.25, heartbeat_timeout: 1..2,
               listen: {}, advertise: {}, metrics_registry: Prometheus::Client::Registry.new)
	@join_hints, @keys, @logger, @heartbeat_interval, @heartbeat_timeout = join_hints, shared_keys, logger, heartbeat_interval, heartbeat_timeout

	@metrics = Evinrude::Metrics.new(metrics_registry)

	@listen, @advertise = listen, advertise
	@listen[:address] ||= "::"
	@listen[:port]    ||= 0

	if storage_dir
		@storage_dir = Pathname.new(storage_dir)
	end

	snapshot = if @storage_dir
		if !@storage_dir.exist?
			@storage_dir.mkdir
		end

		if !@storage_dir.directory?
			raise ArgumentError, "Storage directory #{@storage_dir} isn't *actually* a directory"
		end

		snapshot_file = @storage_dir.join("snapshot.yaml")

		if snapshot_file.exist?
			@metrics.snapshot_file_size.set(snapshot_file.stat.size)
			YAML.load_file(snapshot_file)
		end
	end

	@state_machine_class = state_machine

	if snapshot
		@node_name        = snapshot.node_name
		@state_machine    = @state_machine_class.new(snapshot: snapshot.state)
		@last_command_ids = snapshot.last_command_ids
	else
		@node_name        = node_name || SecureRandom.uuid
		@state_machine    = @state_machine_class.new
		@last_command_ids = {}
	end

	@sm_mutex      = Mutex.new

	if snapshot
		@config        = snapshot.cluster_config
		@config_index  = snapshot.cluster_config_index
		@config.metrics = @metrics
		@config.logger  = logger
	else
		@config       = Evinrude::ClusterConfiguration.new(logger: logger, metrics: @metrics)
		@config_index = 0
	end

	@last_append  = Time.at(0)
	@current_term = 0
	@voted_for    = nil
	@mode         = :init

	@metrics.term.set(0)

	if snapshot
		logger.debug(logloc) { "Configuring log from snapshot; snapshot_last_term=#{snapshot.last_term} snapshot_last_index=#{snapshot.last_index}" }
		@log = Evinrude::Log.new(snapshot_last_term: snapshot.last_term, snapshot_last_index: snapshot.last_index, logger: logger)
	else
		@log = Evinrude::Log.new(logger: logger)
	end

	if snapshot
		logger.debug(logloc) { "Setting commit_index to #{snapshot.last_index} from snapshot" }
		@commit_index = snapshot.last_index
	else
		@commit_index = 0
	end

	@metrics.commit_index.set(@commit_index)

	@peers = Hash.new do |h, k|
		backoff = Evinrude::Backoff.new

		peer_conn = @network.connect(address: k.address, port: k.port)

		h[k] = Peer.new(metrics: @metrics, conn: peer_conn, node_info: k, next_index: @log.last_index + 1)
	end

	@config_change_queue = []
	@config_change_request_in_progress = nil
	@cc_sem = Async::Semaphore.new
end

Instance Attribute Details

#node_name ⇒ Object (readonly)

Returns the value of attribute node_name.



20
21
22
 
# File 'lib/evinrude.rb', line 20

def node_name
  @node_name
end

Instance Method Details

#address ⇒ Object 



208
209
210
 
# File 'lib/evinrude.rb', line 208

def address
	@network&.advertised_address
end

#candidate? ⇒ Boolean

Returns:

  • (Boolean) 


228
229
230
 
# File 'lib/evinrude.rb', line 228

def candidate?
	@mode == :candidate
end

#command(s) ⇒ Object 



115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/evinrude.rb', line 115

def command(s)
	@metrics.command_execution.measure do
		Async(logger: logger) do |task|
			command_id = SecureRandom.uuid

			loop do
				reply = rpc_to_leader(Message::CommandRequest.new(command: s, id: command_id, node_name: @node_name), task)

				if reply.success
					break true
				end
			end
		end.result
	end
end

#expired? ⇒ Boolean

Returns:

  • (Boolean) 


236
237
238
 
# File 'lib/evinrude.rb', line 236

def expired?
	!!(!leader? && @heartbeat_timeout_time && @heartbeat_timeout_time < Time.now)
end

#follower? ⇒ Boolean

Returns:

  • (Boolean) 


224
225
226
 
# File 'lib/evinrude.rb', line 224

def follower?
	@mode == :follower
end

#init? ⇒ Boolean

Returns:

  • (Boolean) 


232
233
234
 
# File 'lib/evinrude.rb', line 232

def init?
	@mode == :init
end

#leader? ⇒ Boolean

Returns:

  • (Boolean) 


220
221
222
 
# File 'lib/evinrude.rb', line 220

def leader?
	@mode == :leader
end

#node_info ⇒ Object 



240
241
242
243
244
245
246
 
# File 'lib/evinrude.rb', line 240

def node_info
	if @network.nil?
		raise RuntimeError, "Cannot determine node info until the network is up"
	end

	@node_info ||= Evinrude::NodeInfo.new(address: address, port: port, name: @node_name)
end

#nodes ⇒ Object 



216
217
218
 
# File 'lib/evinrude.rb', line 216

def nodes
	@config.nodes
end

#port ⇒ Object 



212
213
214
 
# File 'lib/evinrude.rb', line 212

def port
	@network&.advertised_port
end

#remove_node(node_info, unsafe: false) ⇒ Object 



186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
 
# File 'lib/evinrude.rb', line 186

def remove_node(node_info, unsafe: false)
	if unsafe
		logger.warn(logloc) { "Unsafely removing node #{node_info.inspect} from the local configuration" }

		@config.remove_node(node_info, force: true)
	else
		@metrics.remove_node.measure do
			Async(logger: logger) do |task|
				loop do
					logger.debug(logloc) { "(in #{@node_name}) Requesting removal of #{node_info.inspect}" }

					reply = rpc_to_leader(Evinrude::Message::NodeRemovalRequest.new(node_info: node_info, unsafe: unsafe), task)

					if reply.success
						break true
					end
				end
			end.result
		end
	end
end

#run ⇒ Object 



157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
 
# File 'lib/evinrude.rb', line 157

def run
	logger.info(logloc) { "Evinrude node #{@node_name} starting up" }

	@metrics.start_time.set(Time.now.to_f)

	if @storage_dir
		@metrics.log_loaded_from_disk.set(1)
		load_log_from_disk
	else
		@metrics.log_loaded_from_disk.set(0)
	end

	Async do |task|  #(logger: logger) do |task|
		@async_task = task
		@network = Network.new(keys: @keys, logger: logger, metrics: @metrics, listen: @listen, advertise: @advertise).start

		logger.info(logloc) { "Node #{@node_name} listening on #{address}:#{port}" }

		@metrics.info.set(1, labels: { node_name: @node_name, listen_address: @network.listen_address, listen_port: @network.listen_port, advertise_address: address, advertise_port: port })

		task.async { process_rpc_requests }

		join_or_create_cluster
	end.return
rescue => ex
	log_exception(ex) { "Fatal error" }
	raise
end

#state ⇒ Object 



131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/evinrude.rb', line 131

def state
	@metrics.read_state.measure do
		Async(logger: logger) do |task|
			loop do
				state_object = nil
				commit_index = nil

				@sm_mutex.synchronize do
					# Disturbingly, this appears to be one of the best available ways
					# to make a guaranteed deep copy of an arbitrary object
					state_object = YAML.load(@state_machine.current_state.to_yaml)
					commit_index = @commit_index
				end

				logger.debug(logloc) { "(in #{@node_name}) Checking if #{state_object.inspect} at commit_index=#{commit_index} is the most up-to-date state" }

				reply = rpc_to_leader(Evinrude::Message::ReadRequest.new(commit_index: commit_index), task)

				if reply.success
					break state_object
				end
			end
		end.result
	end
end