Module: Gitlab::GitalyClient

Defined in:
lib/gitlab/gitaly_client.rb,
lib/gitlab/gitaly_client/call.rb,
lib/gitlab/gitaly_client/diff.rb,
lib/gitlab/gitaly_client/util.rb,
lib/gitlab/gitaly_client/wiki_page.rb,
lib/gitlab/gitaly_client/ref_service.rb,
lib/gitlab/gitaly_client/blob_service.rb,
lib/gitlab/gitaly_client/diff_stitcher.rb,
lib/gitlab/gitaly_client/attributes_bag.rb,
lib/gitlab/gitaly_client/blobs_stitcher.rb,
lib/gitlab/gitaly_client/commit_service.rb,
lib/gitlab/gitaly_client/remote_service.rb,
lib/gitlab/gitaly_client/server_service.rb,
lib/gitlab/gitaly_client/cleanup_service.rb,
lib/gitlab/gitaly_client/queue_enumerator.rb,
lib/gitlab/gitaly_client/storage_settings.rb,
lib/gitlab/gitaly_client/conflicts_service.rb,
lib/gitlab/gitaly_client/namespace_service.rb,
lib/gitlab/gitaly_client/operation_service.rb,
lib/gitlab/gitaly_client/list_blobs_adapter.rb,
lib/gitlab/gitaly_client/repository_service.rb,
lib/gitlab/gitaly_client/object_pool_service.rb,
lib/gitlab/gitaly_client/health_check_service.rb,
lib/gitlab/gitaly_client/praefect_info_service.rb,
lib/gitlab/gitaly_client/conflict_files_stitcher.rb,
lib/gitlab/gitaly_client/with_feature_flag_actors.rb

Defined Under Namespace

Modules: AttributesBag, Util, WithFeatureFlagActors Classes: BlobService, BlobsStitcher, Call, CleanupService, CommitService, ConflictFilesStitcher, ConflictsService, Diff, DiffStitcher, HealthCheckService, ListBlobsAdapter, NamespaceService, ObjectPoolService, OperationService, PraefectInfoService, QueueEnumerator, RefService, RemoteService, RepositoryService, ServerService, StorageSettings, TooManyInvocationsError, WikiPage

Constant Summary collapse

SERVER_VERSION_FILE = 
'GITALY_SERVER_VERSION'
MAXIMUM_GITALY_CALLS = 
30
CLIENT_NAME = 
(Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
GITALY_METADATA_FILENAME = 
'.gitaly-metadata'
MUTEX = 
Mutex.new

Class Method Summary collapse

Class Method Details

.add_call_details(details) ⇒ Object 



380
381
382
383
 
# File 'lib/gitlab/gitaly_client.rb', line 380

def self.add_call_details(details)
  Gitlab::SafeRequestStore['gitaly_call_details'] ||= []
  Gitlab::SafeRequestStore['gitaly_call_details'] << details
end

.add_query_time(duration) ⇒ Object 



183
184
185
186
187
188
 
# File 'lib/gitlab/gitaly_client.rb', line 183

def self.add_query_time(duration)
  return unless Gitlab::SafeRequestStore.active?

  Gitlab::SafeRequestStore[:gitaly_query_time] ||= 0
  Gitlab::SafeRequestStore[:gitaly_query_time] += duration
end

.address(storage) ⇒ Object 



112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/gitlab/gitaly_client.rb', line 112

def self.address(storage)
  params = Gitlab.config.repositories.storages[storage]
  raise "storage not found: #{storage.inspect}" if params.nil?

  address = params['gitaly_address']
  unless address.present?
    raise "storage #{storage.inspect} is missing a gitaly_address"
  end

  unless %w(tcp unix tls).include?(URI(address).scheme)
    raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'"
  end

  address
end

.address_metadata(storage) ⇒ Object 



128
129
130
 
# File 'lib/gitlab/gitaly_client.rb', line 128

def self.address_metadata(storage)
  Base64.strict_encode64(Gitlab::Json.dump(storage => connection_data(storage)))
end

.allow_n_plus_1_calls ⇒ Object 



321
322
323
324
325
326
327
328
329
330
 
# File 'lib/gitlab/gitaly_client.rb', line 321

def self.allow_n_plus_1_calls
  return yield unless Gitlab::SafeRequestStore.active?

  begin
    increment_call_count(:gitaly_call_count_exception_block_depth)
    yield
  ensure
    decrement_call_count(:gitaly_call_count_exception_block_depth)
  end
end

.allow_ref_name_caching ⇒ Object

Normally a FindCommit RPC will cache the commit with its SHA instead of a ref name, since it’s possible the branch is mutated afterwards. However, for read-only requests that never mutate the branch, this method allows caching of the ref name directly.



336
337
338
339
340
341
342
343
344
345
346
 
# File 'lib/gitlab/gitaly_client.rb', line 336

def self.allow_ref_name_caching
  return yield unless Gitlab::SafeRequestStore.active?
  return yield if ref_name_caching_allowed?

  begin
    Gitlab::SafeRequestStore[:allow_ref_name_caching] = true
    yield
  ensure
    Gitlab::SafeRequestStore[:allow_ref_name_caching] = false
  end
end

.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block) ⇒ Object

All Gitaly RPC call sites should use GitalyClient.call. This method makes sure that per-request authentication headers are set.

This method optionally takes a block which receives the keyword arguments hash ‘kwargs’ that will be passed to gRPC. This allows the caller to modify or augment the keyword arguments. The block must return a hash.

For example:

GitalyClient.call(storage, service, rpc, request) do |kwargs|

kwargs.merge(deadline: Time.now + 10)

end

The optional remote_storage keyword argument is used to enable inter-gitaly calls. Say you have an RPC that needs to pull data from one repository to another. For example, to fetch a branch from a (non-deduplicated) fork into the fork parent. In that case you would send an RPC call to the Gitaly server hosting the fork parent, and in the request, you would tell that Gitaly server to pull Git data from the fork. How does that Gitaly server connect to the Gitaly server the forked repo lives on? This is the problem ‘remote_storage:` solves: it adds address and authentication information to the call, as gRPC metadata (under the `gitaly-servers` header). The request would say “pull from repo X on gitaly-2”. In the Ruby code you pass `remote_storage: ’gitaly-2’‘. And then the metadata would say “gitaly-2 is at network address tcp://10.0.1.2:8075”.



164
165
166
 
# File 'lib/gitlab/gitaly_client.rb', line 164

def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block)
  Gitlab::GitalyClient::Call.new(storage, service, rpc, request, remote_storage, timeout).call(&block)
end

.can_use_disk?(storage) ⇒ Boolean

Returns:

  • (Boolean) 


429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
 
# File 'lib/gitlab/gitaly_client.rb', line 429

def self.can_use_disk?(storage)
  cached_value = MUTEX.synchronize do
    @can_use_disk ||= {}
    @can_use_disk[storage]
  end

  return cached_value unless cached_value.nil?

  gitaly_filesystem_id = filesystem_id(storage)
  direct_filesystem_id = filesystem_id_from_disk(storage)

  MUTEX.synchronize do
    @can_use_disk[storage] = gitaly_filesystem_id.present? &&
      gitaly_filesystem_id == direct_filesystem_id
  end
end

.clear_stubs! ⇒ Object 



102
103
104
105
106
 
# File 'lib/gitlab/gitaly_client.rb', line 102

def self.clear_stubs!
  MUTEX.synchronize do
    @stubs = nil
  end
end

.connection_data(storage) ⇒ Object 



132
133
134
 
# File 'lib/gitlab/gitaly_client.rb', line 132

def self.connection_data(storage)
  { 'address' => address(storage), 'token' => token(storage) }
end

.decode_detailed_error(err) ⇒ Object 



508
509
510
511
512
513
514
515
516
517
518
519
520
521
 
# File 'lib/gitlab/gitaly_client.rb', line 508

def self.decode_detailed_error(err)
  # details could have more than one in theory, but we only have one to worry about for now.
  detailed_error = err.to_rpc_status&.details&.first

  return unless detailed_error.present?

  prefix = %r{type\.googleapis\.com\/gitaly\.(?<error_type>.+)}
  error_type = prefix.match(detailed_error.type_url)[:error_type]

  Gitaly.const_get(error_type, false).decode(detailed_error.value)
rescue NameError, NoMethodError
  # Error Class might not be known to ruby yet
  nil
end

.default_timeout ⇒ Object

The default timeout on all Gitaly calls



401
402
403
 
# File 'lib/gitlab/gitaly_client.rb', line 401

def self.default_timeout
  timeout(:gitaly_timeout_default)
end

.enforce_gitaly_request_limits(call_site) ⇒ Object

Ensures that Gitaly is not being abuse through n+1 misuse etc

Raises:



282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
 
# File 'lib/gitlab/gitaly_client.rb', line 282

def self.enforce_gitaly_request_limits(call_site)
  # Only count limits in request-response environments
  return unless Gitlab::SafeRequestStore.active?

  # This is this actual number of times this call was made. Used for information purposes only
  actual_call_count = increment_call_count("gitaly_#{call_site}_actual")

  return unless enforce_gitaly_request_limits?

  # Check if this call is nested within a allow_n_plus_1_calls
  # block and skip check if it is
  return if get_call_count(:gitaly_call_count_exception_block_depth) > 0

  # This is the count of calls outside of a `allow_n_plus_1_calls` block
  # It is used for enforcement but not statistics
  permitted_call_count = increment_call_count("gitaly_#{call_site}_permitted")

  count_stack

  return if permitted_call_count <= MAXIMUM_GITALY_CALLS

  raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks)
end

.execute(storage, service, rpc, request, remote_storage:, timeout:) ⇒ Object 



168
169
170
171
172
173
174
175
176
 
# File 'lib/gitlab/gitaly_client.rb', line 168

def self.execute(storage, service, rpc, request, remote_storage:, timeout:)
  enforce_gitaly_request_limits(:call)
  Gitlab::RequestContext.instance.ensure_deadline_not_exceeded!

  kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage)
  kwargs = yield(kwargs) if block_given?

  stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
end

.expected_server_version ⇒ Object 



391
392
393
394
 
# File 'lib/gitlab/gitaly_client.rb', line 391

def self.expected_server_version
  path = Rails.root.join(SERVER_VERSION_FILE)
  path.read.chomp
end

.fast_timeout ⇒ Object 



405
406
407
 
# File 'lib/gitlab/gitaly_client.rb', line 405

def self.fast_timeout
  timeout(:gitaly_timeout_fast)
end

.feature_flag_actors ⇒ Object 



536
537
538
539
540
541
542
 
# File 'lib/gitlab/gitaly_client.rb', line 536

def self.feature_flag_actors
  if Gitlab::SafeRequestStore.active?
    Gitlab::SafeRequestStore[:gitaly_feature_flag_actors] ||= {}
  else
    Thread.current[:gitaly_feature_flag_actors] ||= {}
  end
end

.filesystem_disk_available(storage) ⇒ Object 



458
459
460
 
# File 'lib/gitlab/gitaly_client.rb', line 458

def self.filesystem_disk_available(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.available
end

.filesystem_disk_used(storage) ⇒ Object 



462
463
464
 
# File 'lib/gitlab/gitaly_client.rb', line 462

def self.filesystem_disk_used(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.used
end

.filesystem_id(storage) ⇒ Object 



446
447
448
 
# File 'lib/gitlab/gitaly_client.rb', line 446

def self.filesystem_id(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_info&.filesystem_id
end

.filesystem_id_from_disk(storage) ⇒ Object 



450
451
452
453
454
455
456
 
# File 'lib/gitlab/gitaly_client.rb', line 450

def self.filesystem_id_from_disk(storage)
  metadata_file = File.read(storage_metadata_file_path(storage))
  metadata_hash = Gitlab::Json.parse(metadata_file)
  metadata_hash['gitaly_filesystem_id']
rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError
  nil
end

.get_request_count ⇒ Object

Returns the of the number of Gitaly calls made for this request



369
370
371
 
# File 'lib/gitlab/gitaly_client.rb', line 369

def self.get_request_count
  get_call_count("gitaly_call_actual")
end

.list_call_details ⇒ Object 



385
386
387
388
389
 
# File 'lib/gitlab/gitaly_client.rb', line 385

def self.list_call_details
  return [] unless Gitlab::PerformanceBar.enabled_for_request?

  Gitlab::SafeRequestStore['gitaly_call_details'] || []
end

.long_timeout ⇒ Object 



413
414
415
416
417
418
419
 
# File 'lib/gitlab/gitaly_client.rb', line 413

def self.long_timeout
  if Gitlab::Runtime.puma?
    default_timeout
  else
    6.hours
  end
end

.medium_timeout ⇒ Object 



409
410
411
 
# File 'lib/gitlab/gitaly_client.rb', line 409

def self.medium_timeout
  timeout(:gitaly_timeout_medium)
end

.query_time ⇒ Object 



178
179
180
181
 
# File 'lib/gitlab/gitaly_client.rb', line 178

def self.query_time
  query_time = Gitlab::SafeRequestStore[:gitaly_query_time] || 0
  query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION)
end

.random_storage ⇒ Object 



108
109
110
 
# File 'lib/gitlab/gitaly_client.rb', line 108

def self.random_storage
  Gitlab.config.repositories.storages.keys.sample
end

.ref_name_caching_allowed? ⇒ Boolean

Returns:

  • (Boolean) 


348
349
350
 
# File 'lib/gitlab/gitaly_client.rb', line 348

def self.ref_name_caching_allowed?
  Gitlab::SafeRequestStore[:allow_ref_name_caching]
end

.request_kwargs(storage, timeout:, remote_storage: nil) ⇒ Object 



210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/gitlab/gitaly_client.rb', line 210

def self.request_kwargs(storage, timeout:, remote_storage: nil)
  metadata = {
    'authorization' => "Bearer #{authorization_token(storage)}",
    'client_name' => CLIENT_NAME
  }

  context_data = Gitlab::ApplicationContext.current

  feature_stack = Thread.current[:gitaly_feature_stack]
  feature = feature_stack && feature_stack[0]
  metadata['call_site'] = feature.to_s if feature
  metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
  metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id
  metadata['gitaly-session-id'] = session_id
  metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil)
  metadata['user_id'] = context_data['meta.user_id'].to_s if context_data&.fetch('meta.user_id', nil)
  metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil)
  metadata.merge!(Feature::Gitaly.server_feature_flags(**feature_flag_actors))
  metadata.merge!(route_to_primary)

  deadline_info = request_deadline(timeout)
  metadata.merge!(deadline_info.slice(:deadline_type))

  { metadata: metadata, deadline: deadline_info[:deadline] }
end

.reset_counts ⇒ Object 



373
374
375
376
377
378
 
# File 'lib/gitlab/gitaly_client.rb', line 373

def self.reset_counts
  return unless Gitlab::SafeRequestStore.active?

  Gitlab::SafeRequestStore["gitaly_call_actual"] = 0
  Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0
end

.session_id ⇒ Object 



270
271
272
 
# File 'lib/gitlab/gitaly_client.rb', line 270

def self.session_id
  Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid
end

.storage_metadata_file_path(storage) ⇒ Object 



421
422
423
424
425
426
427
 
# File 'lib/gitlab/gitaly_client.rb', line 421

def self.storage_metadata_file_path(storage)
  Gitlab::GitalyClient::StorageSettings.allow_disk_access do
    File.join(
      Gitlab.config.repositories.storages[storage].legacy_disk_path, GITALY_METADATA_FILENAME
    )
  end
end

.stub(name, storage) ⇒ Object 



34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/gitlab/gitaly_client.rb', line 34

def self.stub(name, storage)
  MUTEX.synchronize do
    @stubs ||= {}
    @stubs[storage] ||= {}
    @stubs[storage][name] ||= begin
      klass = stub_class(name)
      addr = stub_address(storage)
      creds = stub_creds(storage)
      klass.new(addr, creds, interceptors: interceptors, channel_args: channel_args)
    end
  end
end

.stub_address(storage) ⇒ Object 



98
99
100
 
# File 'lib/gitlab/gitaly_client.rb', line 98

def self.stub_address(storage)
  address(storage).sub(%r{^tcp://|^tls://}, '')
end

.stub_class(name) ⇒ Object 



90
91
92
93
94
95
96
 
# File 'lib/gitlab/gitaly_client.rb', line 90

def self.stub_class(name)
  if name == :health_check
    Grpc::Health::V1::Health::Stub
  else
    Gitaly.const_get(name.to_s.camelcase.to_sym, false).const_get(:Stub, false)
  end
end

.stub_creds(storage) ⇒ Object 



82
83
84
85
86
87
88
 
# File 'lib/gitlab/gitaly_client.rb', line 82

def self.stub_creds(storage)
  if URI(address(storage)).scheme == 'tls'
    GRPC::Core::ChannelCredentials.new ::Gitlab::X509::Certificate.ca_certs_bundle
  else
    :this_channel_is_insecure
  end
end

.timestamp(time) ⇒ Object 



396
397
398
 
# File 'lib/gitlab/gitaly_client.rb', line 396

def self.timestamp(time)
  Google::Protobuf::Timestamp.new(seconds: time.to_i)
end

.token(storage) ⇒ Object 



274
275
276
277
278
279
 
# File 'lib/gitlab/gitaly_client.rb', line 274

def self.token(storage)
  params = Gitlab.config.repositories.storages[storage]
  raise "storage not found: #{storage.inspect}" if params.nil?

  params['gitaly_token'].presence || Gitlab.config.gitaly['token']
end

.with_feature_flag_actors(repository: nil, user: nil, project: nil, group: nil, &block) ⇒ Object 



525
526
527
528
529
530
531
532
533
534
 
# File 'lib/gitlab/gitaly_client.rb', line 525

def self.with_feature_flag_actors(repository: nil, user: nil, project: nil, group: nil, &block)
  feature_flag_actors[:repository] = repository
  feature_flag_actors[:user] = user
  feature_flag_actors[:project] = project
  feature_flag_actors[:group] = group

  yield
ensure
  feature_flag_actors.clear
end