Module: Gitlab::GitalyClient
- Defined in:
- lib/gitlab/gitaly_client.rb,
lib/gitlab/gitaly_client/call.rb,
lib/gitlab/gitaly_client/diff.rb,
lib/gitlab/gitaly_client/util.rb,
lib/gitlab/gitaly_client/wiki_file.rb,
lib/gitlab/gitaly_client/wiki_page.rb,
lib/gitlab/gitaly_client/ref_service.rb,
lib/gitlab/gitaly_client/blob_service.rb,
lib/gitlab/gitaly_client/wiki_service.rb,
lib/gitlab/gitaly_client/diff_stitcher.rb,
lib/gitlab/gitaly_client/attributes_bag.rb,
lib/gitlab/gitaly_client/blobs_stitcher.rb,
lib/gitlab/gitaly_client/commit_service.rb,
lib/gitlab/gitaly_client/remote_service.rb,
lib/gitlab/gitaly_client/server_service.rb,
lib/gitlab/gitaly_client/cleanup_service.rb,
lib/gitlab/gitaly_client/queue_enumerator.rb,
lib/gitlab/gitaly_client/storage_settings.rb,
lib/gitlab/gitaly_client/conflicts_service.rb,
lib/gitlab/gitaly_client/namespace_service.rb,
lib/gitlab/gitaly_client/operation_service.rb,
lib/gitlab/gitaly_client/repository_service.rb,
lib/gitlab/gitaly_client/object_pool_service.rb,
lib/gitlab/gitaly_client/health_check_service.rb,
lib/gitlab/gitaly_client/praefect_info_service.rb,
lib/gitlab/gitaly_client/conflict_files_stitcher.rb
Defined Under Namespace
Modules: AttributesBag, Util
Classes: BlobService, BlobsStitcher, Call, CleanupService, CommitService, ConflictFilesStitcher, ConflictsService, Diff, DiffStitcher, HealthCheckService, NamespaceService, ObjectPoolService, OperationService, PraefectInfoService, QueueEnumerator, RefService, RemoteService, RepositoryService, ServerService, StorageSettings, TooManyInvocationsError, WikiFile, WikiPage, WikiService
Constant Summary
collapse
- PEM_REGEX =
/\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze
- SERVER_VERSION_FILE =
'GITALY_SERVER_VERSION'
- MAXIMUM_GITALY_CALLS =
30
- CLIENT_NAME =
(Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
- GITALY_METADATA_FILENAME =
'.gitaly-metadata'
- MUTEX =
Mutex.new
Class Method Summary
collapse
-
.add_call_details(details) ⇒ Object
-
.add_query_time(duration) ⇒ Object
-
.address(storage) ⇒ Object
-
.address_metadata(storage) ⇒ Object
-
.allow_n_plus_1_calls ⇒ Object
-
.allow_ref_name_caching ⇒ Object
Normally a FindCommit RPC will cache the commit with its SHA instead of a ref name, since it's possible the branch is mutated afterwards.
-
.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block) ⇒ Object
All Gitaly RPC call sites should use GitalyClient.call.
-
.can_use_disk?(storage) ⇒ Boolean
-
.clear_stubs! ⇒ Object
-
.connection_data(storage) ⇒ Object
-
.default_timeout ⇒ Object
The default timeout on all Gitaly calls.
-
.enforce_gitaly_request_limits(call_site) ⇒ Object
Ensures that Gitaly is not being abuse through n+1 misuse etc.
-
.execute(storage, service, rpc, request, remote_storage:, timeout:) ⇒ Object
-
.expected_server_version ⇒ Object
-
.fast_timeout ⇒ Object
-
.filesystem_disk_available(storage) ⇒ Object
-
.filesystem_disk_used(storage) ⇒ Object
-
.filesystem_id(storage) ⇒ Object
-
.filesystem_id_from_disk(storage) ⇒ Object
-
.get_request_count ⇒ Object
Returns the of the number of Gitaly calls made for this request.
-
.list_call_details ⇒ Object
-
.long_timeout ⇒ Object
-
.medium_timeout ⇒ Object
-
.query_time ⇒ Object
-
.random_storage ⇒ Object
-
.ref_name_caching_allowed? ⇒ Boolean
-
.request_kwargs(storage, timeout:, remote_storage: nil) ⇒ Object
-
.reset_counts ⇒ Object
-
.session_id ⇒ Object
-
.storage_metadata_file_path(storage) ⇒ Object
-
.stub(name, storage) ⇒ Object
-
.stub_address(storage) ⇒ Object
-
.stub_cert_paths ⇒ Object
-
.stub_certs ⇒ Object
-
.stub_class(name) ⇒ Object
-
.stub_creds(storage) ⇒ Object
-
.timestamp(time) ⇒ Object
-
.token(storage) ⇒ Object
Class Method Details
.add_call_details(details) ⇒ Object
.add_query_time(duration) ⇒ Object
.address(storage) ⇒ Object
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
# File 'lib/gitlab/gitaly_client.rb', line 114
def self.address(storage)
params = Gitlab.config.repositories.storages[storage]
raise "storage not found: #{storage.inspect}" if params.nil?
address = params['gitaly_address']
unless address.present?
raise "storage #{storage.inspect} is missing a gitaly_address"
end
unless URI(address).scheme.in?(%w(tcp unix tls))
raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'"
end
address
end
|
130
131
132
|
# File 'lib/gitlab/gitaly_client.rb', line 130
def self.address_metadata(storage)
Base64.strict_encode64(Gitlab::Json.dump(storage => connection_data(storage)))
end
|
.allow_n_plus_1_calls ⇒ Object
297
298
299
300
301
302
303
304
305
306
|
# File 'lib/gitlab/gitaly_client.rb', line 297
def self.allow_n_plus_1_calls
return yield unless Gitlab::SafeRequestStore.active?
begin
increment_call_count(:gitaly_call_count_exception_block_depth)
yield
ensure
decrement_call_count(:gitaly_call_count_exception_block_depth)
end
end
|
.allow_ref_name_caching ⇒ Object
Normally a FindCommit RPC will cache the commit with its SHA instead of a ref name, since it's possible the branch is mutated afterwards. However, for read-only requests that never mutate the branch, this method allows caching of the ref name directly.
312
313
314
315
316
317
318
319
320
321
322
|
# File 'lib/gitlab/gitaly_client.rb', line 312
def self.allow_ref_name_caching
return yield unless Gitlab::SafeRequestStore.active?
return yield if ref_name_caching_allowed?
begin
Gitlab::SafeRequestStore[:allow_ref_name_caching] = true
yield
ensure
Gitlab::SafeRequestStore[:allow_ref_name_caching] = false
end
end
|
.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block) ⇒ Object
All Gitaly RPC call sites should use GitalyClient.call. This method makes sure that per-request authentication headers are set.
This method optionally takes a block which receives the keyword arguments hash 'kwargs' that will be passed to gRPC. This allows the caller to modify or augment the keyword arguments. The block must return a hash.
For example:
GitalyClient.call(storage, service, rpc, request) do |kwargs|
kwargs.merge(deadline: Time.now + 10)
end
The optional remote_storage keyword argument is used to enable inter-gitaly calls. Say you have an RPC that needs to pull data from one repository to another. For example, to fetch a branch from a (non-deduplicated) fork into the fork parent. In that case you would send an RPC call to the Gitaly server hosting the fork parent, and in the request, you would tell that Gitaly server to pull Git data from the fork. How does that Gitaly server connect to the Gitaly server the forked repo lives on? This is the problem `remote_storage:` solves: it adds address and authentication information to the call, as gRPC metadata (under the `gitaly-servers` header). The request would say “pull from repo X on gitaly-2”. In the Ruby code you pass `remote_storage: 'gitaly-2'`. And then the metadata would say “gitaly-2 is at network address tcp://10.0.1.2:8075”.
166
167
168
|
# File 'lib/gitlab/gitaly_client.rb', line 166
def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block)
Gitlab::GitalyClient::Call.new(storage, service, rpc, request, remote_storage, timeout).call(&block)
end
|
.can_use_disk?(storage) ⇒ Boolean
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
|
# File 'lib/gitlab/gitaly_client.rb', line 405
def self.can_use_disk?(storage)
cached_value = MUTEX.synchronize do
@can_use_disk ||= {}
@can_use_disk[storage]
end
return cached_value unless cached_value.nil?
gitaly_filesystem_id = filesystem_id(storage)
direct_filesystem_id = filesystem_id_from_disk(storage)
MUTEX.synchronize do
@can_use_disk[storage] = gitaly_filesystem_id.present? &&
gitaly_filesystem_id == direct_filesystem_id
end
end
|
.clear_stubs! ⇒ Object
104
105
106
107
108
|
# File 'lib/gitlab/gitaly_client.rb', line 104
def self.clear_stubs!
MUTEX.synchronize do
@stubs = nil
end
end
|
.connection_data(storage) ⇒ Object
134
135
136
|
# File 'lib/gitlab/gitaly_client.rb', line 134
def self.connection_data(storage)
{ 'address' => address(storage), 'token' => token(storage) }
end
|
.default_timeout ⇒ Object
The default timeout on all Gitaly calls
377
378
379
|
# File 'lib/gitlab/gitaly_client.rb', line 377
def self.default_timeout
timeout(:gitaly_timeout_default)
end
|
.enforce_gitaly_request_limits(call_site) ⇒ Object
Ensures that Gitaly is not being abuse through n+1 misuse etc
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
|
# File 'lib/gitlab/gitaly_client.rb', line 260
def self.enforce_gitaly_request_limits(call_site)
return unless Gitlab::SafeRequestStore.active?
actual_call_count = increment_call_count("gitaly_#{call_site}_actual")
return unless enforce_gitaly_request_limits?
return if get_call_count(:gitaly_call_count_exception_block_depth) > 0
permitted_call_count = increment_call_count("gitaly_#{call_site}_permitted")
count_stack
return if permitted_call_count <= MAXIMUM_GITALY_CALLS
raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks)
end
|
.execute(storage, service, rpc, request, remote_storage:, timeout:) ⇒ Object
170
171
172
173
174
175
176
177
178
|
# File 'lib/gitlab/gitaly_client.rb', line 170
def self.execute(storage, service, rpc, request, remote_storage:, timeout:)
enforce_gitaly_request_limits(:call)
Gitlab::RequestContext.instance.ensure_deadline_not_exceeded!
kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage)
kwargs = yield(kwargs) if block_given?
stub(service, storage).__send__(rpc, request, kwargs) end
|
.expected_server_version ⇒ Object
367
368
369
370
|
# File 'lib/gitlab/gitaly_client.rb', line 367
def self.expected_server_version
path = Rails.root.join(SERVER_VERSION_FILE)
path.read.chomp
end
|
.fast_timeout ⇒ Object
381
382
383
|
# File 'lib/gitlab/gitaly_client.rb', line 381
def self.fast_timeout
timeout(:gitaly_timeout_fast)
end
|
.filesystem_disk_available(storage) ⇒ Object
.filesystem_disk_used(storage) ⇒ Object
.filesystem_id(storage) ⇒ Object
.filesystem_id_from_disk(storage) ⇒ Object
426
427
428
429
430
431
432
|
# File 'lib/gitlab/gitaly_client.rb', line 426
def self.filesystem_id_from_disk(storage)
metadata_file = File.read(storage_metadata_file_path(storage))
metadata_hash = Gitlab::Json.parse(metadata_file)
metadata_hash['gitaly_filesystem_id']
rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError
nil
end
|
.get_request_count ⇒ Object
Returns the of the number of Gitaly calls made for this request
345
346
347
|
# File 'lib/gitlab/gitaly_client.rb', line 345
def self.get_request_count
get_call_count("gitaly_call_actual")
end
|
.list_call_details ⇒ Object
.long_timeout ⇒ Object
389
390
391
392
393
394
395
|
# File 'lib/gitlab/gitaly_client.rb', line 389
def self.long_timeout
if Gitlab::Runtime.web_server?
default_timeout
else
6.hours
end
end
|
.medium_timeout ⇒ Object
385
386
387
|
# File 'lib/gitlab/gitaly_client.rb', line 385
def self.medium_timeout
timeout(:gitaly_timeout_medium)
end
|
.random_storage ⇒ Object
110
111
112
|
# File 'lib/gitlab/gitaly_client.rb', line 110
def self.random_storage
Gitlab.config.repositories.storages.keys.sample
end
|
.ref_name_caching_allowed? ⇒ Boolean
324
325
326
|
# File 'lib/gitlab/gitaly_client.rb', line 324
def self.ref_name_caching_allowed?
Gitlab::SafeRequestStore[:allow_ref_name_caching]
end
|
.request_kwargs(storage, timeout:, remote_storage: nil) ⇒ Object
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
|
# File 'lib/gitlab/gitaly_client.rb', line 212
def self.request_kwargs(storage, timeout:, remote_storage: nil)
metadata = {
'authorization' => "Bearer #{authorization_token(storage)}",
'client_name' => CLIENT_NAME
}
feature_stack = Thread.current[:gitaly_feature_stack]
feature = feature_stack && feature_stack[0]
metadata['call_site'] = feature.to_s if feature
metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id
metadata['gitaly-session-id'] = session_id
metadata.merge!(Feature::Gitaly.server_feature_flags)
deadline_info = request_deadline(timeout)
metadata.merge!(deadline_info.slice(:deadline_type))
{ metadata: metadata, deadline: deadline_info[:deadline] }
end
|
.session_id ⇒ Object
248
249
250
|
# File 'lib/gitlab/gitaly_client.rb', line 248
def self.session_id
Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid
end
|
.stub(name, storage) ⇒ Object
35
36
37
38
39
40
41
42
43
44
45
46
|
# File 'lib/gitlab/gitaly_client.rb', line 35
def self.stub(name, storage)
MUTEX.synchronize do
@stubs ||= {}
@stubs[storage] ||= {}
@stubs[storage][name] ||= begin
klass = stub_class(name)
addr = stub_address(storage)
creds = stub_creds(storage)
klass.new(addr, creds, interceptors: interceptors, channel_args: channel_args)
end
end
end
|
.stub_address(storage) ⇒ Object
100
101
102
|
# File 'lib/gitlab/gitaly_client.rb', line 100
def self.stub_address(storage)
address(storage).sub(%r{^tcp://|^tls://}, '')
end
|
.stub_cert_paths ⇒ Object
65
66
67
68
69
|
# File 'lib/gitlab/gitaly_client.rb', line 65
def self.stub_cert_paths
cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"]
cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE
cert_paths
end
|
.stub_certs ⇒ Object
71
72
73
74
75
76
77
78
79
80
81
82
|
# File 'lib/gitlab/gitaly_client.rb', line 71
def self.stub_certs
return @certs if @certs
@certs = stub_cert_paths.flat_map do |cert_file|
File.read(cert_file).scan(PEM_REGEX).map do |cert|
OpenSSL::X509::Certificate.new(cert).to_pem
rescue OpenSSL::OpenSSLError => e
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, cert_file: cert_file)
nil
end.compact
end.uniq.join("\n")
end
|
.stub_class(name) ⇒ Object
92
93
94
95
96
97
98
|
# File 'lib/gitlab/gitaly_client.rb', line 92
def self.stub_class(name)
if name == :health_check
Grpc::Health::V1::Health::Stub
else
Gitaly.const_get(name.to_s.camelcase.to_sym, false).const_get(:Stub, false)
end
end
|
.stub_creds(storage) ⇒ Object
84
85
86
87
88
89
90
|
# File 'lib/gitlab/gitaly_client.rb', line 84
def self.stub_creds(storage)
if URI(address(storage)).scheme == 'tls'
GRPC::Core::ChannelCredentials.new stub_certs
else
:this_channel_is_insecure
end
end
|
.timestamp(time) ⇒ Object
372
373
374
|
# File 'lib/gitlab/gitaly_client.rb', line 372
def self.timestamp(time)
Google::Protobuf::Timestamp.new(seconds: time.to_i)
end
|
.token(storage) ⇒ Object
252
253
254
255
256
257
|
# File 'lib/gitlab/gitaly_client.rb', line 252
def self.token(storage)
params = Gitlab.config.repositories.storages[storage]
raise "storage not found: #{storage.inspect}" if params.nil?
params['gitaly_token'].presence || Gitlab.config.gitaly['token']
end
|