Module: Gitlab::GitalyClient

Defined in:
lib/gitlab/gitaly_client.rb,
lib/gitlab/gitaly_client/call.rb,
lib/gitlab/gitaly_client/diff.rb,
lib/gitlab/gitaly_client/util.rb,
lib/gitlab/gitaly_client/wiki_page.rb,
lib/gitlab/gitaly_client/ref_service.rb,
lib/gitlab/gitaly_client/blob_service.rb,
lib/gitlab/gitaly_client/diff_stitcher.rb,
lib/gitlab/gitaly_client/attributes_bag.rb,
lib/gitlab/gitaly_client/blobs_stitcher.rb,
lib/gitlab/gitaly_client/commit_service.rb,
lib/gitlab/gitaly_client/remote_service.rb,
lib/gitlab/gitaly_client/server_service.rb,
lib/gitlab/gitaly_client/cleanup_service.rb,
lib/gitlab/gitaly_client/queue_enumerator.rb,
lib/gitlab/gitaly_client/storage_settings.rb,
lib/gitlab/gitaly_client/conflicts_service.rb,
lib/gitlab/gitaly_client/namespace_service.rb,
lib/gitlab/gitaly_client/operation_service.rb,
lib/gitlab/gitaly_client/list_blobs_adapter.rb,
lib/gitlab/gitaly_client/repository_service.rb,
lib/gitlab/gitaly_client/object_pool_service.rb,
lib/gitlab/gitaly_client/health_check_service.rb,
lib/gitlab/gitaly_client/praefect_info_service.rb,
lib/gitlab/gitaly_client/conflict_files_stitcher.rb,
lib/gitlab/gitaly_client/with_feature_flag_actors.rb

Defined Under Namespace

Modules: AttributesBag, Util, WithFeatureFlagActors Classes: BlobService, BlobsStitcher, Call, CleanupService, CommitService, ConflictFilesStitcher, ConflictsService, Diff, DiffStitcher, HealthCheckService, ListBlobsAdapter, NamespaceService, ObjectPoolService, OperationService, PraefectInfoService, QueueEnumerator, RefService, RemoteService, RepositoryService, ServerService, StorageSettings, TooManyInvocationsError, WikiPage

Constant Summary collapse

SERVER_VERSION_FILE = 
'GITALY_SERVER_VERSION'
MAXIMUM_GITALY_CALLS = 
30
CLIENT_NAME = 
(Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
GITALY_METADATA_FILENAME = 
'.gitaly-metadata'
MUTEX = 
Mutex.new

Class Method Summary collapse

Class Method Details

.add_call_details(details) ⇒ Object 



390
391
392
393
 
# File 'lib/gitlab/gitaly_client.rb', line 390

def self.add_call_details(details)
  Gitlab::SafeRequestStore['gitaly_call_details'] ||= []
  Gitlab::SafeRequestStore['gitaly_call_details'] << details
end

.add_query_time(duration) ⇒ Object 



193
194
195
196
197
198
 
# File 'lib/gitlab/gitaly_client.rb', line 193

def self.add_query_time(duration)
  return unless Gitlab::SafeRequestStore.active?

  Gitlab::SafeRequestStore[:gitaly_query_time] ||= 0
  Gitlab::SafeRequestStore[:gitaly_query_time] += duration
end

.address(storage) ⇒ Object 



122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
 
# File 'lib/gitlab/gitaly_client.rb', line 122

def self.address(storage)
  params = Gitlab.config.repositories.storages[storage]
  raise "storage not found: #{storage.inspect}" if params.nil?

  address = params['gitaly_address']
  unless address.present?
    raise "storage #{storage.inspect} is missing a gitaly_address"
  end

  unless %w[tcp unix tls dns].include?(URI(address).scheme)
    raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls' or 'dns'"
  end

  address
end

.address_metadata(storage) ⇒ Object 



138
139
140
 
# File 'lib/gitlab/gitaly_client.rb', line 138

def self.address_metadata(storage)
  Base64.strict_encode64(Gitlab::Json.dump(storage => connection_data(storage)))
end

.allow_n_plus_1_calls ⇒ Object 



331
332
333
334
335
336
337
338
339
340
 
# File 'lib/gitlab/gitaly_client.rb', line 331

def self.allow_n_plus_1_calls
  return yield unless Gitlab::SafeRequestStore.active?

  begin
    increment_call_count(:gitaly_call_count_exception_block_depth)
    yield
  ensure
    decrement_call_count(:gitaly_call_count_exception_block_depth)
  end
end

.allow_ref_name_caching ⇒ Object

Normally a FindCommit RPC will cache the commit with its SHA instead of a ref name, since it’s possible the branch is mutated afterwards. However, for read-only requests that never mutate the branch, this method allows caching of the ref name directly.



346
347
348
349
350
351
352
353
354
355
356
 
# File 'lib/gitlab/gitaly_client.rb', line 346

def self.allow_ref_name_caching
  return yield unless Gitlab::SafeRequestStore.active?
  return yield if ref_name_caching_allowed?

  begin
    Gitlab::SafeRequestStore[:allow_ref_name_caching] = true
    yield
  ensure
    Gitlab::SafeRequestStore[:allow_ref_name_caching] = false
  end
end

.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block) ⇒ Object

All Gitaly RPC call sites should use GitalyClient.call. This method makes sure that per-request authentication headers are set.

This method optionally takes a block which receives the keyword arguments hash ‘kwargs’ that will be passed to gRPC. This allows the caller to modify or augment the keyword arguments. The block must return a hash.

For example:

GitalyClient.call(storage, service, rpc, request) do |kwargs|

kwargs.merge(deadline: Time.now + 10)

end

The optional remote_storage keyword argument is used to enable inter-gitaly calls. Say you have an RPC that needs to pull data from one repository to another. For example, to fetch a branch from a (non-deduplicated) fork into the fork parent. In that case you would send an RPC call to the Gitaly server hosting the fork parent, and in the request, you would tell that Gitaly server to pull Git data from the fork. How does that Gitaly server connect to the Gitaly server the forked repo lives on? This is the problem ‘remote_storage:` solves: it adds address and authentication information to the call, as gRPC metadata (under the `gitaly-servers` header). The request would say “pull from repo X on gitaly-2”. In the Ruby code you pass `remote_storage: ’gitaly-2’‘. And then the metadata would say “gitaly-2 is at network address tcp://10.0.1.2:8075”.



174
175
176
 
# File 'lib/gitlab/gitaly_client.rb', line 174

def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block)
  Gitlab::GitalyClient::Call.new(storage, service, rpc, request, remote_storage, timeout).call(&block)
end

.can_use_disk?(storage) ⇒ Boolean

Returns:

  • (Boolean) 


439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
 
# File 'lib/gitlab/gitaly_client.rb', line 439

def self.can_use_disk?(storage)
  cached_value = MUTEX.synchronize do
    @can_use_disk ||= {}
    @can_use_disk[storage]
  end

  return cached_value unless cached_value.nil?

  gitaly_filesystem_id = filesystem_id(storage)
  direct_filesystem_id = filesystem_id_from_disk(storage)

  MUTEX.synchronize do
    @can_use_disk[storage] = gitaly_filesystem_id.present? &&
      gitaly_filesystem_id == direct_filesystem_id
  end
end

.clear_stubs! ⇒ Object 



111
112
113
114
115
116
 
# File 'lib/gitlab/gitaly_client.rb', line 111

def self.clear_stubs!
  MUTEX.synchronize do
    @stubs = nil
    @channels = nil
  end
end

.connection_data(storage) ⇒ Object 



142
143
144
 
# File 'lib/gitlab/gitaly_client.rb', line 142

def self.connection_data(storage)
  { 'address' => address(storage), 'token' => token(storage) }
end

.create_channel(storage) ⇒ Object

Cache gRPC servers by storage. All the client stubs in the same process can share the underlying connection to the same host thanks to HTTP2 framing protocol that gRPC is built on top. This method is not thread-safe. It is intended to be a part of ‘stub`, method behind a mutex protection.



104
105
106
107
108
109
 
# File 'lib/gitlab/gitaly_client.rb', line 104

def self.create_channel(storage)
  @channels ||= {}
  @channels[storage] ||= GRPC::ClientStub.setup_channel(
    nil, stub_address(storage), stub_creds(storage), channel_args
  )
end

.decode_detailed_error(err) ⇒ Object 



518
519
520
521
522
523
524
525
526
527
528
529
530
531
 
# File 'lib/gitlab/gitaly_client.rb', line 518

def self.decode_detailed_error(err)
  # details could have more than one in theory, but we only have one to worry about for now.
  detailed_error = err.to_rpc_status&.details&.first

  return unless detailed_error.present?

  prefix = %r{type\.googleapis\.com\/gitaly\.(?<error_type>.+)}
  error_type = prefix.match(detailed_error.type_url)[:error_type]

  Gitaly.const_get(error_type, false).decode(detailed_error.value)
rescue NameError, NoMethodError
  # Error Class might not be known to ruby yet
  nil
end

.default_timeout ⇒ Object

The default timeout on all Gitaly calls



411
412
413
 
# File 'lib/gitlab/gitaly_client.rb', line 411

def self.default_timeout
  timeout(:gitaly_timeout_default)
end

.enforce_gitaly_request_limits(call_site) ⇒ Object

Ensures that Gitaly is not being abuse through n+1 misuse etc

Raises:



292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
 
# File 'lib/gitlab/gitaly_client.rb', line 292

def self.enforce_gitaly_request_limits(call_site)
  # Only count limits in request-response environments
  return unless Gitlab::SafeRequestStore.active?

  # This is this actual number of times this call was made. Used for information purposes only
  actual_call_count = increment_call_count("gitaly_#{call_site}_actual")

  return unless enforce_gitaly_request_limits?

  # Check if this call is nested within a allow_n_plus_1_calls
  # block and skip check if it is
  return if get_call_count(:gitaly_call_count_exception_block_depth) > 0

  # This is the count of calls outside of a `allow_n_plus_1_calls` block
  # It is used for enforcement but not statistics
  permitted_call_count = increment_call_count("gitaly_#{call_site}_permitted")

  count_stack

  return if permitted_call_count <= MAXIMUM_GITALY_CALLS

  raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks)
end

.execute(storage, service, rpc, request, remote_storage:, timeout:) ⇒ Object 



178
179
180
181
182
183
184
185
186
 
# File 'lib/gitlab/gitaly_client.rb', line 178

def self.execute(storage, service, rpc, request, remote_storage:, timeout:)
  enforce_gitaly_request_limits(:call)
  Gitlab::RequestContext.instance.ensure_deadline_not_exceeded!

  kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage)
  kwargs = yield(kwargs) if block_given?

  stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
end

.expected_server_version ⇒ Object 



401
402
403
404
 
# File 'lib/gitlab/gitaly_client.rb', line 401

def self.expected_server_version
  path = Rails.root.join(SERVER_VERSION_FILE)
  path.read.chomp
end

.fast_timeout ⇒ Object 



415
416
417
 
# File 'lib/gitlab/gitaly_client.rb', line 415

def self.fast_timeout
  timeout(:gitaly_timeout_fast)
end

.feature_flag_actors ⇒ Object 



546
547
548
549
550
551
552
 
# File 'lib/gitlab/gitaly_client.rb', line 546

def self.feature_flag_actors
  if Gitlab::SafeRequestStore.active?
    Gitlab::SafeRequestStore[:gitaly_feature_flag_actors] ||= {}
  else
    Thread.current[:gitaly_feature_flag_actors] ||= {}
  end
end

.filesystem_disk_available(storage) ⇒ Object 



468
469
470
 
# File 'lib/gitlab/gitaly_client.rb', line 468

def self.filesystem_disk_available(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.available
end

.filesystem_disk_used(storage) ⇒ Object 



472
473
474
 
# File 'lib/gitlab/gitaly_client.rb', line 472

def self.filesystem_disk_used(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.used
end

.filesystem_id(storage) ⇒ Object 



456
457
458
 
# File 'lib/gitlab/gitaly_client.rb', line 456

def self.filesystem_id(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_info&.filesystem_id
end

.filesystem_id_from_disk(storage) ⇒ Object 



460
461
462
463
464
465
466
 
# File 'lib/gitlab/gitaly_client.rb', line 460

def self.filesystem_id_from_disk(storage)
  metadata_file = File.read(storage_metadata_file_path(storage))
  metadata_hash = Gitlab::Json.parse(metadata_file)
  metadata_hash['gitaly_filesystem_id']
rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError
  nil
end

.get_request_count ⇒ Object

Returns the of the number of Gitaly calls made for this request



379
380
381
 
# File 'lib/gitlab/gitaly_client.rb', line 379

def self.get_request_count
  get_call_count("gitaly_call_actual")
end

.list_call_details ⇒ Object 



395
396
397
398
399
 
# File 'lib/gitlab/gitaly_client.rb', line 395

def self.list_call_details
  return [] unless Gitlab::PerformanceBar.enabled_for_request?

  Gitlab::SafeRequestStore['gitaly_call_details'] || []
end

.long_timeout ⇒ Object 



423
424
425
426
427
428
429
 
# File 'lib/gitlab/gitaly_client.rb', line 423

def self.long_timeout
  if Gitlab::Runtime.puma?
    default_timeout
  else
    6.hours
  end
end

.medium_timeout ⇒ Object 



419
420
421
 
# File 'lib/gitlab/gitaly_client.rb', line 419

def self.medium_timeout
  timeout(:gitaly_timeout_medium)
end

.query_time ⇒ Object 



188
189
190
191
 
# File 'lib/gitlab/gitaly_client.rb', line 188

def self.query_time
  query_time = Gitlab::SafeRequestStore[:gitaly_query_time] || 0
  query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION)
end

.random_storage ⇒ Object 



118
119
120
 
# File 'lib/gitlab/gitaly_client.rb', line 118

def self.random_storage
  Gitlab.config.repositories.storages.keys.sample
end

.ref_name_caching_allowed? ⇒ Boolean

Returns:

  • (Boolean) 


358
359
360
 
# File 'lib/gitlab/gitaly_client.rb', line 358

def self.ref_name_caching_allowed?
  Gitlab::SafeRequestStore[:allow_ref_name_caching]
end

.request_kwargs(storage, timeout:, remote_storage: nil) ⇒ Object 



220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
 
# File 'lib/gitlab/gitaly_client.rb', line 220

def self.request_kwargs(storage, timeout:, remote_storage: nil)
  metadata = {
    'authorization' => "Bearer #{authorization_token(storage)}",
    'client_name' => CLIENT_NAME
  }

  context_data = Gitlab::ApplicationContext.current

  feature_stack = Thread.current[:gitaly_feature_stack]
  feature = feature_stack && feature_stack[0]
  metadata['call_site'] = feature.to_s if feature
  metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
  metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id
  metadata['gitaly-session-id'] = session_id
  metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil)
  metadata['user_id'] = context_data['meta.user_id'].to_s if context_data&.fetch('meta.user_id', nil)
  metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil)
  metadata.merge!(Feature::Gitaly.server_feature_flags(**feature_flag_actors))
  metadata.merge!(route_to_primary)

  deadline_info = request_deadline(timeout)
  metadata.merge!(deadline_info.slice(:deadline_type))

  { metadata: metadata, deadline: deadline_info[:deadline] }
end

.reset_counts ⇒ Object 



383
384
385
386
387
388
 
# File 'lib/gitlab/gitaly_client.rb', line 383

def self.reset_counts
  return unless Gitlab::SafeRequestStore.active?

  Gitlab::SafeRequestStore["gitaly_call_actual"] = 0
  Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0
end

.session_id ⇒ Object 



280
281
282
 
# File 'lib/gitlab/gitaly_client.rb', line 280

def self.session_id
  Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid
end

.storage_metadata_file_path(storage) ⇒ Object 



431
432
433
434
435
436
437
 
# File 'lib/gitlab/gitaly_client.rb', line 431

def self.storage_metadata_file_path(storage)
  Gitlab::GitalyClient::StorageSettings.allow_disk_access do
    File.join(
      Gitlab.config.repositories.storages[storage].legacy_disk_path, GITALY_METADATA_FILENAME
    )
  end
end

.stub(name, storage) ⇒ Object 



34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/gitlab/gitaly_client.rb', line 34

def self.stub(name, storage)
  MUTEX.synchronize do
    @stubs ||= {}
    @stubs[storage] ||= {}
    @stubs[storage][name] ||= begin
      klass = stub_class(name)
      channel = create_channel(storage)
      klass.new(channel.target, nil, interceptors: interceptors, channel_override: channel)
    end
  end
end

.stub_address(storage) ⇒ Object 



97
98
99
 
# File 'lib/gitlab/gitaly_client.rb', line 97

def self.stub_address(storage)
  address(storage).sub(%r{^tcp://|^tls://}, '')
end

.stub_class(name) ⇒ Object 



89
90
91
92
93
94
95
 
# File 'lib/gitlab/gitaly_client.rb', line 89

def self.stub_class(name)
  if name == :health_check
    Grpc::Health::V1::Health::Stub
  else
    Gitaly.const_get(name.to_s.camelcase.to_sym, false).const_get(:Stub, false)
  end
end

.stub_creds(storage) ⇒ Object 



81
82
83
84
85
86
87
 
# File 'lib/gitlab/gitaly_client.rb', line 81

def self.stub_creds(storage)
  if URI(address(storage)).scheme == 'tls'
    GRPC::Core::ChannelCredentials.new ::Gitlab::X509::Certificate.ca_certs_bundle
  else
    :this_channel_is_insecure
  end
end

.timestamp(time) ⇒ Object 



406
407
408
 
# File 'lib/gitlab/gitaly_client.rb', line 406

def self.timestamp(time)
  Google::Protobuf::Timestamp.new(seconds: time.to_i)
end

.token(storage) ⇒ Object 



284
285
286
287
288
289
 
# File 'lib/gitlab/gitaly_client.rb', line 284

def self.token(storage)
  params = Gitlab.config.repositories.storages[storage]
  raise "storage not found: #{storage.inspect}" if params.nil?

  params['gitaly_token'].presence || Gitlab.config.gitaly['token']
end

.with_feature_flag_actors(repository: nil, user: nil, project: nil, group: nil, &block) ⇒ Object 



535
536
537
538
539
540
541
542
543
544
 
# File 'lib/gitlab/gitaly_client.rb', line 535

def self.with_feature_flag_actors(repository: nil, user: nil, project: nil, group: nil, &block)
  feature_flag_actors[:repository] = repository
  feature_flag_actors[:user] = user
  feature_flag_actors[:project] = project
  feature_flag_actors[:group] = group

  yield
ensure
  feature_flag_actors.clear
end