Module: Gitlab::GitalyClient

Defined in:
lib/gitlab/gitaly_client.rb,
lib/gitlab/gitaly_client/call.rb,
lib/gitlab/gitaly_client/diff.rb,
lib/gitlab/gitaly_client/util.rb,
lib/gitlab/gitaly_client/wiki_page.rb,
lib/gitlab/gitaly_client/ref_service.rb,
lib/gitlab/gitaly_client/blob_service.rb,
lib/gitlab/gitaly_client/wiki_service.rb,
lib/gitlab/gitaly_client/diff_stitcher.rb,
lib/gitlab/gitaly_client/attributes_bag.rb,
lib/gitlab/gitaly_client/blobs_stitcher.rb,
lib/gitlab/gitaly_client/commit_service.rb,
lib/gitlab/gitaly_client/remote_service.rb,
lib/gitlab/gitaly_client/server_service.rb,
lib/gitlab/gitaly_client/cleanup_service.rb,
lib/gitlab/gitaly_client/queue_enumerator.rb,
lib/gitlab/gitaly_client/storage_settings.rb,
lib/gitlab/gitaly_client/conflicts_service.rb,
lib/gitlab/gitaly_client/namespace_service.rb,
lib/gitlab/gitaly_client/operation_service.rb,
lib/gitlab/gitaly_client/list_blobs_adapter.rb,
lib/gitlab/gitaly_client/repository_service.rb,
lib/gitlab/gitaly_client/object_pool_service.rb,
lib/gitlab/gitaly_client/health_check_service.rb,
lib/gitlab/gitaly_client/praefect_info_service.rb,
lib/gitlab/gitaly_client/conflict_files_stitcher.rb

Defined Under Namespace

Modules: AttributesBag, Util Classes: BlobService, BlobsStitcher, Call, CleanupService, CommitService, ConflictFilesStitcher, ConflictsService, Diff, DiffStitcher, HealthCheckService, ListBlobsAdapter, NamespaceService, ObjectPoolService, OperationService, PraefectInfoService, QueueEnumerator, RefService, RemoteService, RepositoryService, ServerService, StorageSettings, TooManyInvocationsError, WikiPage, WikiService

Constant Summary collapse

PEM_REGEX = 
/\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze
SERVER_VERSION_FILE = 
'GITALY_SERVER_VERSION'
MAXIMUM_GITALY_CALLS = 
30
CLIENT_NAME = 
(Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
GITALY_METADATA_FILENAME = 
'.gitaly-metadata'
MUTEX = 
Mutex.new

Class Method Summary collapse

Class Method Details

.add_call_details(details) ⇒ Object 


379
380
381
382
 
# File 'lib/gitlab/gitaly_client.rb', line 379

def self.add_call_details(details)
  Gitlab::SafeRequestStore['gitaly_call_details'] ||= []
  Gitlab::SafeRequestStore['gitaly_call_details'] << details
end

.add_query_time(duration) ⇒ Object 


185
186
187
188
189
190
 
# File 'lib/gitlab/gitaly_client.rb', line 185

def self.add_query_time(duration)
  return unless Gitlab::SafeRequestStore.active?

  Gitlab::SafeRequestStore[:gitaly_query_time] ||= 0
  Gitlab::SafeRequestStore[:gitaly_query_time] += duration
end

.address(storage) ⇒ Object 


114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/gitlab/gitaly_client.rb', line 114

def self.address(storage)
  params = Gitlab.config.repositories.storages[storage]
  raise "storage not found: #{storage.inspect}" if params.nil?

  address = params['gitaly_address']
  unless address.present?
    raise "storage #{storage.inspect} is missing a gitaly_address"
  end

  unless %w(tcp unix tls).include?(URI(address).scheme)
    raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'"
  end

  address
end

.address_metadata(storage) ⇒ Object 


130
131
132
 
# File 'lib/gitlab/gitaly_client.rb', line 130

def self.address_metadata(storage)
  Base64.strict_encode64(Gitlab::Json.dump(storage => connection_data(storage)))
end

.allow_n_plus_1_calls ⇒ Object 


320
321
322
323
324
325
326
327
328
329
 
# File 'lib/gitlab/gitaly_client.rb', line 320

def self.allow_n_plus_1_calls
  return yield unless Gitlab::SafeRequestStore.active?

  begin
    increment_call_count(:gitaly_call_count_exception_block_depth)
    yield
  ensure
    decrement_call_count(:gitaly_call_count_exception_block_depth)
  end
end

.allow_ref_name_caching ⇒ Object

Normally a FindCommit RPC will cache the commit with its SHA instead of a ref name, since it's possible the branch is mutated afterwards. However, for read-only requests that never mutate the branch, this method allows caching of the ref name directly.


335
336
337
338
339
340
341
342
343
344
345
 
# File 'lib/gitlab/gitaly_client.rb', line 335

def self.allow_ref_name_caching
  return yield unless Gitlab::SafeRequestStore.active?
  return yield if ref_name_caching_allowed?

  begin
    Gitlab::SafeRequestStore[:allow_ref_name_caching] = true
    yield
  ensure
    Gitlab::SafeRequestStore[:allow_ref_name_caching] = false
  end
end

.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block) ⇒ Object

All Gitaly RPC call sites should use GitalyClient.call. This method makes sure that per-request authentication headers are set.

This method optionally takes a block which receives the keyword arguments hash 'kwargs' that will be passed to gRPC. This allows the caller to modify or augment the keyword arguments. The block must return a hash.

For example:

GitalyClient.call(storage, service, rpc, request) do |kwargs|

kwargs.merge(deadline: Time.now + 10)

end

The optional remote_storage keyword argument is used to enable inter-gitaly calls. Say you have an RPC that needs to pull data from one repository to another. For example, to fetch a branch from a (non-deduplicated) fork into the fork parent. In that case you would send an RPC call to the Gitaly server hosting the fork parent, and in the request, you would tell that Gitaly server to pull Git data from the fork. How does that Gitaly server connect to the Gitaly server the forked repo lives on? This is the problem `remote_storage:` solves: it adds address and authentication information to the call, as gRPC metadata (under the `gitaly-servers` header). The request would say “pull from repo X on gitaly-2”. In the Ruby code you pass `remote_storage: 'gitaly-2'`. And then the metadata would say “gitaly-2 is at network address tcp://10.0.1.2:8075”.


166
167
168
 
# File 'lib/gitlab/gitaly_client.rb', line 166

def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block)
  Gitlab::GitalyClient::Call.new(storage, service, rpc, request, remote_storage, timeout).call(&block)
end

.can_use_disk?(storage) ⇒ Boolean

Returns:

  • (Boolean) 

428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
 
# File 'lib/gitlab/gitaly_client.rb', line 428

def self.can_use_disk?(storage)
  cached_value = MUTEX.synchronize do
    @can_use_disk ||= {}
    @can_use_disk[storage]
  end

  return cached_value unless cached_value.nil?

  gitaly_filesystem_id = filesystem_id(storage)
  direct_filesystem_id = filesystem_id_from_disk(storage)

  MUTEX.synchronize do
    @can_use_disk[storage] = gitaly_filesystem_id.present? &&
      gitaly_filesystem_id == direct_filesystem_id
  end
end

.clear_stubs! ⇒ Object 


104
105
106
107
108
 
# File 'lib/gitlab/gitaly_client.rb', line 104

def self.clear_stubs!
  MUTEX.synchronize do
    @stubs = nil
  end
end

.connection_data(storage) ⇒ Object 


134
135
136
 
# File 'lib/gitlab/gitaly_client.rb', line 134

def self.connection_data(storage)
  { 'address' => address(storage), 'token' => token(storage) }
end

.default_timeout ⇒ Object

The default timeout on all Gitaly calls


400
401
402
 
# File 'lib/gitlab/gitaly_client.rb', line 400

def self.default_timeout
  timeout(:gitaly_timeout_default)
end

.enforce_gitaly_request_limits(call_site) ⇒ Object

Ensures that Gitaly is not being abuse through n+1 misuse etc

Raises:


283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
 
# File 'lib/gitlab/gitaly_client.rb', line 283

def self.enforce_gitaly_request_limits(call_site)
  # Only count limits in request-response environments
  return unless Gitlab::SafeRequestStore.active?

  # This is this actual number of times this call was made. Used for information purposes only
  actual_call_count = increment_call_count("gitaly_#{call_site}_actual")

  return unless enforce_gitaly_request_limits?

  # Check if this call is nested within a allow_n_plus_1_calls
  # block and skip check if it is
  return if get_call_count(:gitaly_call_count_exception_block_depth) > 0

  # This is the count of calls outside of a `allow_n_plus_1_calls` block
  # It is used for enforcement but not statistics
  permitted_call_count = increment_call_count("gitaly_#{call_site}_permitted")

  count_stack

  return if permitted_call_count <= MAXIMUM_GITALY_CALLS

  raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks)
end

.execute(storage, service, rpc, request, remote_storage:, timeout:) ⇒ Object 


170
171
172
173
174
175
176
177
178
 
# File 'lib/gitlab/gitaly_client.rb', line 170

def self.execute(storage, service, rpc, request, remote_storage:, timeout:)
  enforce_gitaly_request_limits(:call)
  Gitlab::RequestContext.instance.ensure_deadline_not_exceeded!

  kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage)
  kwargs = yield(kwargs) if block_given?

  stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
end

.expected_server_version ⇒ Object 


390
391
392
393
 
# File 'lib/gitlab/gitaly_client.rb', line 390

def self.expected_server_version
  path = Rails.root.join(SERVER_VERSION_FILE)
  path.read.chomp
end

.fast_timeout ⇒ Object 


404
405
406
 
# File 'lib/gitlab/gitaly_client.rb', line 404

def self.fast_timeout
  timeout(:gitaly_timeout_fast)
end

.filesystem_disk_available(storage) ⇒ Object 


457
458
459
 
# File 'lib/gitlab/gitaly_client.rb', line 457

def self.filesystem_disk_available(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.available
end

.filesystem_disk_used(storage) ⇒ Object 


461
462
463
 
# File 'lib/gitlab/gitaly_client.rb', line 461

def self.filesystem_disk_used(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.used
end

.filesystem_id(storage) ⇒ Object 


445
446
447
 
# File 'lib/gitlab/gitaly_client.rb', line 445

def self.filesystem_id(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_info&.filesystem_id
end

.filesystem_id_from_disk(storage) ⇒ Object 


449
450
451
452
453
454
455
 
# File 'lib/gitlab/gitaly_client.rb', line 449

def self.filesystem_id_from_disk(storage)
  metadata_file = File.read(storage_metadata_file_path(storage))
  metadata_hash = Gitlab::Json.parse(metadata_file)
  metadata_hash['gitaly_filesystem_id']
rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError
  nil
end

.get_request_count ⇒ Object

Returns the of the number of Gitaly calls made for this request


368
369
370
 
# File 'lib/gitlab/gitaly_client.rb', line 368

def self.get_request_count
  get_call_count("gitaly_call_actual")
end

.list_call_details ⇒ Object 


384
385
386
387
388
 
# File 'lib/gitlab/gitaly_client.rb', line 384

def self.list_call_details
  return [] unless Gitlab::PerformanceBar.enabled_for_request?

  Gitlab::SafeRequestStore['gitaly_call_details'] || []
end

.long_timeout ⇒ Object 


412
413
414
415
416
417
418
 
# File 'lib/gitlab/gitaly_client.rb', line 412

def self.long_timeout
  if Gitlab::Runtime.web_server?
    default_timeout
  else
    6.hours
  end
end

.medium_timeout ⇒ Object 


408
409
410
 
# File 'lib/gitlab/gitaly_client.rb', line 408

def self.medium_timeout
  timeout(:gitaly_timeout_medium)
end

.query_time ⇒ Object 


180
181
182
183
 
# File 'lib/gitlab/gitaly_client.rb', line 180

def self.query_time
  query_time = Gitlab::SafeRequestStore[:gitaly_query_time] || 0
  query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION)
end

.random_storage ⇒ Object 


110
111
112
 
# File 'lib/gitlab/gitaly_client.rb', line 110

def self.random_storage
  Gitlab.config.repositories.storages.keys.sample
end

.ref_name_caching_allowed? ⇒ Boolean

Returns:

  • (Boolean) 

347
348
349
 
# File 'lib/gitlab/gitaly_client.rb', line 347

def self.ref_name_caching_allowed?
  Gitlab::SafeRequestStore[:allow_ref_name_caching]
end

.request_kwargs(storage, timeout:, remote_storage: nil) ⇒ Object 


212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
 
# File 'lib/gitlab/gitaly_client.rb', line 212

def self.request_kwargs(storage, timeout:, remote_storage: nil)
  metadata = {
    'authorization' => "Bearer #{authorization_token(storage)}",
    'client_name' => CLIENT_NAME
  }

  context_data = Gitlab::ApplicationContext.current

  feature_stack = Thread.current[:gitaly_feature_stack]
  feature = feature_stack && feature_stack[0]
  metadata['call_site'] = feature.to_s if feature
  metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
  metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id
  metadata['gitaly-session-id'] = session_id
  metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil)
  metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil)
  metadata.merge!(Feature::Gitaly.server_feature_flags)
  metadata.merge!(route_to_primary)

  deadline_info = request_deadline(timeout)
  metadata.merge!(deadline_info.slice(:deadline_type))

  { metadata: metadata, deadline: deadline_info[:deadline] }
end

.reset_counts ⇒ Object 


372
373
374
375
376
377
 
# File 'lib/gitlab/gitaly_client.rb', line 372

def self.reset_counts
  return unless Gitlab::SafeRequestStore.active?

  Gitlab::SafeRequestStore["gitaly_call_actual"] = 0
  Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0
end

.session_id ⇒ Object 


271
272
273
 
# File 'lib/gitlab/gitaly_client.rb', line 271

def self.session_id
  Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid
end

.storage_metadata_file_path(storage) ⇒ Object 


420
421
422
423
424
425
426
 
# File 'lib/gitlab/gitaly_client.rb', line 420

def self.storage_metadata_file_path(storage)
  Gitlab::GitalyClient::StorageSettings.allow_disk_access do
    File.join(
      Gitlab.config.repositories.storages[storage].legacy_disk_path, GITALY_METADATA_FILENAME
    )
  end
end

.stub(name, storage) ⇒ Object 


35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/gitlab/gitaly_client.rb', line 35

def self.stub(name, storage)
  MUTEX.synchronize do
    @stubs ||= {}
    @stubs[storage] ||= {}
    @stubs[storage][name] ||= begin
      klass = stub_class(name)
      addr = stub_address(storage)
      creds = stub_creds(storage)
      klass.new(addr, creds, interceptors: interceptors, channel_args: channel_args)
    end
  end
end

.stub_address(storage) ⇒ Object 


100
101
102
 
# File 'lib/gitlab/gitaly_client.rb', line 100

def self.stub_address(storage)
  address(storage).sub(%r{^tcp://|^tls://}, '')
end

.stub_cert_paths ⇒ Object 


65
66
67
68
69
 
# File 'lib/gitlab/gitaly_client.rb', line 65

def self.stub_cert_paths
  cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"]
  cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE
  cert_paths
end

.stub_certs ⇒ Object 


71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/gitlab/gitaly_client.rb', line 71

def self.stub_certs
  return @certs if @certs

  @certs = stub_cert_paths.flat_map do |cert_file|
    File.read(cert_file).scan(PEM_REGEX).map do |cert|
      OpenSSL::X509::Certificate.new(cert).to_pem
    rescue OpenSSL::OpenSSLError => e
      Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, cert_file: cert_file)
      nil
    end.compact
  end.uniq.join("\n")
end

.stub_class(name) ⇒ Object 


92
93
94
95
96
97
98
 
# File 'lib/gitlab/gitaly_client.rb', line 92

def self.stub_class(name)
  if name == :health_check
    Grpc::Health::V1::Health::Stub
  else
    Gitaly.const_get(name.to_s.camelcase.to_sym, false).const_get(:Stub, false)
  end
end

.stub_creds(storage) ⇒ Object 


84
85
86
87
88
89
90
 
# File 'lib/gitlab/gitaly_client.rb', line 84

def self.stub_creds(storage)
  if URI(address(storage)).scheme == 'tls'
    GRPC::Core::ChannelCredentials.new stub_certs
  else
    :this_channel_is_insecure
  end
end

.timestamp(time) ⇒ Object 


395
396
397
 
# File 'lib/gitlab/gitaly_client.rb', line 395

def self.timestamp(time)
  Google::Protobuf::Timestamp.new(seconds: time.to_i)
end

.token(storage) ⇒ Object 


275
276
277
278
279
280
 
# File 'lib/gitlab/gitaly_client.rb', line 275

def self.token(storage)
  params = Gitlab.config.repositories.storages[storage]
  raise "storage not found: #{storage.inspect}" if params.nil?

  params['gitaly_token'].presence || Gitlab.config.gitaly['token']
end