Module: Gitlab::GitalyClient
- Defined in:
- lib/gitlab/gitaly_client.rb,
lib/gitlab/gitaly_client/call.rb,
lib/gitlab/gitaly_client/diff.rb,
lib/gitlab/gitaly_client/util.rb,
lib/gitlab/gitaly_client/wiki_page.rb,
lib/gitlab/gitaly_client/ref_service.rb,
lib/gitlab/gitaly_client/blob_service.rb,
lib/gitlab/gitaly_client/diff_stitcher.rb,
lib/gitlab/gitaly_client/attributes_bag.rb,
lib/gitlab/gitaly_client/blobs_stitcher.rb,
lib/gitlab/gitaly_client/commit_service.rb,
lib/gitlab/gitaly_client/remote_service.rb,
lib/gitlab/gitaly_client/server_service.rb,
lib/gitlab/gitaly_client/cleanup_service.rb,
lib/gitlab/gitaly_client/queue_enumerator.rb,
lib/gitlab/gitaly_client/storage_settings.rb,
lib/gitlab/gitaly_client/conflicts_service.rb,
lib/gitlab/gitaly_client/namespace_service.rb,
lib/gitlab/gitaly_client/operation_service.rb,
lib/gitlab/gitaly_client/list_blobs_adapter.rb,
lib/gitlab/gitaly_client/repository_service.rb,
lib/gitlab/gitaly_client/object_pool_service.rb,
lib/gitlab/gitaly_client/health_check_service.rb,
lib/gitlab/gitaly_client/praefect_info_service.rb,
lib/gitlab/gitaly_client/conflict_files_stitcher.rb,
lib/gitlab/gitaly_client/with_feature_flag_actors.rb
Defined Under Namespace
Modules: AttributesBag, Util, WithFeatureFlagActors
Classes: BlobService, BlobsStitcher, Call, CleanupService, CommitService, ConflictFilesStitcher, ConflictsService, Diff, DiffStitcher, HealthCheckService, ListBlobsAdapter, NamespaceService, ObjectPoolService, OperationService, PraefectInfoService, QueueEnumerator, RefService, RemoteService, RepositoryService, ServerService, StorageSettings, TooManyInvocationsError, WikiPage
Constant Summary
collapse
- SERVER_VERSION_FILE =
'GITALY_SERVER_VERSION'
- MAXIMUM_GITALY_CALLS =
30
- CLIENT_NAME =
(Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
- GITALY_METADATA_FILENAME =
'.gitaly-metadata'
- MUTEX =
Mutex.new
Class Method Summary
collapse
-
.add_call_details(details) ⇒ Object
-
.add_query_time(duration) ⇒ Object
-
.address(storage) ⇒ Object
-
.address_metadata(storage) ⇒ Object
-
.allow_n_plus_1_calls ⇒ Object
-
.allow_ref_name_caching ⇒ Object
Normally a FindCommit RPC will cache the commit with its SHA instead of a ref name, since it’s possible the branch is mutated afterwards.
-
.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block) ⇒ Object
All Gitaly RPC call sites should use GitalyClient.call.
-
.can_use_disk?(storage) ⇒ Boolean
-
.clear_stubs! ⇒ Object
-
.connection_data(storage) ⇒ Object
-
.decode_detailed_error(err) ⇒ Object
-
.default_timeout ⇒ Object
The default timeout on all Gitaly calls.
-
.enforce_gitaly_request_limits(call_site) ⇒ Object
Ensures that Gitaly is not being abuse through n+1 misuse etc.
-
.execute(storage, service, rpc, request, remote_storage:, timeout:) ⇒ Object
-
.expected_server_version ⇒ Object
-
.fast_timeout ⇒ Object
-
.feature_flag_actors ⇒ Object
-
.filesystem_disk_available(storage) ⇒ Object
-
.filesystem_disk_used(storage) ⇒ Object
-
.filesystem_id(storage) ⇒ Object
-
.filesystem_id_from_disk(storage) ⇒ Object
-
.get_request_count ⇒ Object
Returns the of the number of Gitaly calls made for this request.
-
.list_call_details ⇒ Object
-
.long_timeout ⇒ Object
-
.medium_timeout ⇒ Object
-
.query_time ⇒ Object
-
.random_storage ⇒ Object
-
.ref_name_caching_allowed? ⇒ Boolean
-
.request_kwargs(storage, timeout:, remote_storage: nil) ⇒ Object
-
.reset_counts ⇒ Object
-
.session_id ⇒ Object
-
.storage_metadata_file_path(storage) ⇒ Object
-
.stub(name, storage) ⇒ Object
-
.stub_address(storage) ⇒ Object
-
.stub_class(name) ⇒ Object
-
.stub_creds(storage) ⇒ Object
-
.timestamp(time) ⇒ Object
-
.token(storage) ⇒ Object
-
.with_feature_flag_actors(repository: nil, user: nil, project: nil, group: nil, &block) ⇒ Object
Class Method Details
.add_call_details(details) ⇒ Object
.add_query_time(duration) ⇒ Object
.address(storage) ⇒ Object
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
# File 'lib/gitlab/gitaly_client.rb', line 112
def self.address(storage)
params = Gitlab.config.repositories.storages[storage]
raise "storage not found: #{storage.inspect}" if params.nil?
address = params['gitaly_address']
unless address.present?
raise "storage #{storage.inspect} is missing a gitaly_address"
end
unless %w(tcp unix tls).include?(URI(address).scheme)
raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'"
end
address
end
|
128
129
130
|
# File 'lib/gitlab/gitaly_client.rb', line 128
def self.address_metadata(storage)
Base64.strict_encode64(Gitlab::Json.dump(storage => connection_data(storage)))
end
|
.allow_n_plus_1_calls ⇒ Object
321
322
323
324
325
326
327
328
329
330
|
# File 'lib/gitlab/gitaly_client.rb', line 321
def self.allow_n_plus_1_calls
return yield unless Gitlab::SafeRequestStore.active?
begin
increment_call_count(:gitaly_call_count_exception_block_depth)
yield
ensure
decrement_call_count(:gitaly_call_count_exception_block_depth)
end
end
|
.allow_ref_name_caching ⇒ Object
Normally a FindCommit RPC will cache the commit with its SHA instead of a ref name, since it’s possible the branch is mutated afterwards. However, for read-only requests that never mutate the branch, this method allows caching of the ref name directly.
336
337
338
339
340
341
342
343
344
345
346
|
# File 'lib/gitlab/gitaly_client.rb', line 336
def self.allow_ref_name_caching
return yield unless Gitlab::SafeRequestStore.active?
return yield if ref_name_caching_allowed?
begin
Gitlab::SafeRequestStore[:allow_ref_name_caching] = true
yield
ensure
Gitlab::SafeRequestStore[:allow_ref_name_caching] = false
end
end
|
.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block) ⇒ Object
All Gitaly RPC call sites should use GitalyClient.call. This method makes sure that per-request authentication headers are set.
This method optionally takes a block which receives the keyword arguments hash ‘kwargs’ that will be passed to gRPC. This allows the caller to modify or augment the keyword arguments. The block must return a hash.
For example:
GitalyClient.call(storage, service, rpc, request) do |kwargs|
kwargs.merge(deadline: Time.now + 10)
end
The optional remote_storage keyword argument is used to enable inter-gitaly calls. Say you have an RPC that needs to pull data from one repository to another. For example, to fetch a branch from a (non-deduplicated) fork into the fork parent. In that case you would send an RPC call to the Gitaly server hosting the fork parent, and in the request, you would tell that Gitaly server to pull Git data from the fork. How does that Gitaly server connect to the Gitaly server the forked repo lives on? This is the problem ‘remote_storage:` solves: it adds address and authentication information to the call, as gRPC metadata (under the `gitaly-servers` header). The request would say “pull from repo X on gitaly-2”. In the Ruby code you pass `remote_storage: ’gitaly-2’‘. And then the metadata would say “gitaly-2 is at network address tcp://10.0.1.2:8075”.
164
165
166
|
# File 'lib/gitlab/gitaly_client.rb', line 164
def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block)
Gitlab::GitalyClient::Call.new(storage, service, rpc, request, remote_storage, timeout).call(&block)
end
|
.can_use_disk?(storage) ⇒ Boolean
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
|
# File 'lib/gitlab/gitaly_client.rb', line 429
def self.can_use_disk?(storage)
cached_value = MUTEX.synchronize do
@can_use_disk ||= {}
@can_use_disk[storage]
end
return cached_value unless cached_value.nil?
gitaly_filesystem_id = filesystem_id(storage)
direct_filesystem_id = filesystem_id_from_disk(storage)
MUTEX.synchronize do
@can_use_disk[storage] = gitaly_filesystem_id.present? &&
gitaly_filesystem_id == direct_filesystem_id
end
end
|
.clear_stubs! ⇒ Object
102
103
104
105
106
|
# File 'lib/gitlab/gitaly_client.rb', line 102
def self.clear_stubs!
MUTEX.synchronize do
@stubs = nil
end
end
|
.connection_data(storage) ⇒ Object
132
133
134
|
# File 'lib/gitlab/gitaly_client.rb', line 132
def self.connection_data(storage)
{ 'address' => address(storage), 'token' => token(storage) }
end
|
.decode_detailed_error(err) ⇒ Object
508
509
510
511
512
513
514
515
516
517
518
519
520
521
|
# File 'lib/gitlab/gitaly_client.rb', line 508
def self.decode_detailed_error(err)
detailed_error = err.to_rpc_status&.details&.first
return unless detailed_error.present?
prefix = %r{type\.googleapis\.com\/gitaly\.(?<error_type>.+)}
error_type = prefix.match(detailed_error.type_url)[:error_type]
Gitaly.const_get(error_type, false).decode(detailed_error.value)
rescue NameError, NoMethodError
nil
end
|
.default_timeout ⇒ Object
The default timeout on all Gitaly calls
401
402
403
|
# File 'lib/gitlab/gitaly_client.rb', line 401
def self.default_timeout
timeout(:gitaly_timeout_default)
end
|
.enforce_gitaly_request_limits(call_site) ⇒ Object
Ensures that Gitaly is not being abuse through n+1 misuse etc
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
|
# File 'lib/gitlab/gitaly_client.rb', line 282
def self.enforce_gitaly_request_limits(call_site)
return unless Gitlab::SafeRequestStore.active?
actual_call_count = increment_call_count("gitaly_#{call_site}_actual")
return unless enforce_gitaly_request_limits?
return if get_call_count(:gitaly_call_count_exception_block_depth) > 0
permitted_call_count = increment_call_count("gitaly_#{call_site}_permitted")
count_stack
return if permitted_call_count <= MAXIMUM_GITALY_CALLS
raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks)
end
|
.execute(storage, service, rpc, request, remote_storage:, timeout:) ⇒ Object
168
169
170
171
172
173
174
175
176
|
# File 'lib/gitlab/gitaly_client.rb', line 168
def self.execute(storage, service, rpc, request, remote_storage:, timeout:)
enforce_gitaly_request_limits(:call)
Gitlab::RequestContext.instance.ensure_deadline_not_exceeded!
kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage)
kwargs = yield(kwargs) if block_given?
stub(service, storage).__send__(rpc, request, kwargs) end
|
.expected_server_version ⇒ Object
391
392
393
394
|
# File 'lib/gitlab/gitaly_client.rb', line 391
def self.expected_server_version
path = Rails.root.join(SERVER_VERSION_FILE)
path.read.chomp
end
|
.fast_timeout ⇒ Object
405
406
407
|
# File 'lib/gitlab/gitaly_client.rb', line 405
def self.fast_timeout
timeout(:gitaly_timeout_fast)
end
|
.feature_flag_actors ⇒ Object
536
537
538
539
540
541
542
|
# File 'lib/gitlab/gitaly_client.rb', line 536
def self.feature_flag_actors
if Gitlab::SafeRequestStore.active?
Gitlab::SafeRequestStore[:gitaly_feature_flag_actors] ||= {}
else
Thread.current[:gitaly_feature_flag_actors] ||= {}
end
end
|
.filesystem_disk_available(storage) ⇒ Object
.filesystem_disk_used(storage) ⇒ Object
.filesystem_id(storage) ⇒ Object
.filesystem_id_from_disk(storage) ⇒ Object
450
451
452
453
454
455
456
|
# File 'lib/gitlab/gitaly_client.rb', line 450
def self.filesystem_id_from_disk(storage)
metadata_file = File.read(storage_metadata_file_path(storage))
metadata_hash = Gitlab::Json.parse(metadata_file)
metadata_hash['gitaly_filesystem_id']
rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError
nil
end
|
.get_request_count ⇒ Object
Returns the of the number of Gitaly calls made for this request
369
370
371
|
# File 'lib/gitlab/gitaly_client.rb', line 369
def self.get_request_count
get_call_count("gitaly_call_actual")
end
|
.list_call_details ⇒ Object
.long_timeout ⇒ Object
413
414
415
416
417
418
419
|
# File 'lib/gitlab/gitaly_client.rb', line 413
def self.long_timeout
if Gitlab::Runtime.puma?
default_timeout
else
6.hours
end
end
|
.medium_timeout ⇒ Object
409
410
411
|
# File 'lib/gitlab/gitaly_client.rb', line 409
def self.medium_timeout
timeout(:gitaly_timeout_medium)
end
|
.random_storage ⇒ Object
108
109
110
|
# File 'lib/gitlab/gitaly_client.rb', line 108
def self.random_storage
Gitlab.config.repositories.storages.keys.sample
end
|
.ref_name_caching_allowed? ⇒ Boolean
348
349
350
|
# File 'lib/gitlab/gitaly_client.rb', line 348
def self.ref_name_caching_allowed?
Gitlab::SafeRequestStore[:allow_ref_name_caching]
end
|
.request_kwargs(storage, timeout:, remote_storage: nil) ⇒ Object
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
|
# File 'lib/gitlab/gitaly_client.rb', line 210
def self.request_kwargs(storage, timeout:, remote_storage: nil)
metadata = {
'authorization' => "Bearer #{authorization_token(storage)}",
'client_name' => CLIENT_NAME
}
context_data = Gitlab::ApplicationContext.current
feature_stack = Thread.current[:gitaly_feature_stack]
feature = feature_stack && feature_stack[0]
metadata['call_site'] = feature.to_s if feature
metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id
metadata['gitaly-session-id'] = session_id
metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil)
metadata['user_id'] = context_data['meta.user_id'].to_s if context_data&.fetch('meta.user_id', nil)
metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil)
metadata.merge!(Feature::Gitaly.server_feature_flags(**feature_flag_actors))
metadata.merge!(route_to_primary)
deadline_info = request_deadline(timeout)
metadata.merge!(deadline_info.slice(:deadline_type))
{ metadata: metadata, deadline: deadline_info[:deadline] }
end
|
.session_id ⇒ Object
270
271
272
|
# File 'lib/gitlab/gitaly_client.rb', line 270
def self.session_id
Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid
end
|
.stub(name, storage) ⇒ Object
34
35
36
37
38
39
40
41
42
43
44
45
|
# File 'lib/gitlab/gitaly_client.rb', line 34
def self.stub(name, storage)
MUTEX.synchronize do
@stubs ||= {}
@stubs[storage] ||= {}
@stubs[storage][name] ||= begin
klass = stub_class(name)
addr = stub_address(storage)
creds = stub_creds(storage)
klass.new(addr, creds, interceptors: interceptors, channel_args: channel_args)
end
end
end
|
.stub_address(storage) ⇒ Object
98
99
100
|
# File 'lib/gitlab/gitaly_client.rb', line 98
def self.stub_address(storage)
address(storage).sub(%r{^tcp://|^tls://}, '')
end
|
.stub_class(name) ⇒ Object
90
91
92
93
94
95
96
|
# File 'lib/gitlab/gitaly_client.rb', line 90
def self.stub_class(name)
if name == :health_check
Grpc::Health::V1::Health::Stub
else
Gitaly.const_get(name.to_s.camelcase.to_sym, false).const_get(:Stub, false)
end
end
|
.stub_creds(storage) ⇒ Object
82
83
84
85
86
87
88
|
# File 'lib/gitlab/gitaly_client.rb', line 82
def self.stub_creds(storage)
if URI(address(storage)).scheme == 'tls'
GRPC::Core::ChannelCredentials.new ::Gitlab::X509::Certificate.ca_certs_bundle
else
:this_channel_is_insecure
end
end
|
.timestamp(time) ⇒ Object
396
397
398
|
# File 'lib/gitlab/gitaly_client.rb', line 396
def self.timestamp(time)
Google::Protobuf::Timestamp.new(seconds: time.to_i)
end
|
.token(storage) ⇒ Object
274
275
276
277
278
279
|
# File 'lib/gitlab/gitaly_client.rb', line 274
def self.token(storage)
params = Gitlab.config.repositories.storages[storage]
raise "storage not found: #{storage.inspect}" if params.nil?
params['gitaly_token'].presence || Gitlab.config.gitaly['token']
end
|
.with_feature_flag_actors(repository: nil, user: nil, project: nil, group: nil, &block) ⇒ Object
525
526
527
528
529
530
531
532
533
534
|
# File 'lib/gitlab/gitaly_client.rb', line 525
def self.with_feature_flag_actors(repository: nil, user: nil, project: nil, group: nil, &block)
feature_flag_actors[:repository] = repository
feature_flag_actors[:user] = user
feature_flag_actors[:project] = project
feature_flag_actors[:group] = group
yield
ensure
feature_flag_actors.clear
end
|