Module: Gitlab::GitalyClient

Defined in:
lib/gitlab/gitaly_client.rb,
lib/gitlab/gitaly_client/call.rb,
lib/gitlab/gitaly_client/diff.rb,
lib/gitlab/gitaly_client/util.rb,
lib/gitlab/gitaly_client/wiki_file.rb,
lib/gitlab/gitaly_client/wiki_page.rb,
lib/gitlab/gitaly_client/ref_service.rb,
lib/gitlab/gitaly_client/blob_service.rb,
lib/gitlab/gitaly_client/wiki_service.rb,
lib/gitlab/gitaly_client/diff_stitcher.rb,
lib/gitlab/gitaly_client/attributes_bag.rb,
lib/gitlab/gitaly_client/blobs_stitcher.rb,
lib/gitlab/gitaly_client/commit_service.rb,
lib/gitlab/gitaly_client/remote_service.rb,
lib/gitlab/gitaly_client/server_service.rb,
lib/gitlab/gitaly_client/cleanup_service.rb,
lib/gitlab/gitaly_client/queue_enumerator.rb,
lib/gitlab/gitaly_client/storage_settings.rb,
lib/gitlab/gitaly_client/conflicts_service.rb,
lib/gitlab/gitaly_client/namespace_service.rb,
lib/gitlab/gitaly_client/operation_service.rb,
lib/gitlab/gitaly_client/repository_service.rb,
lib/gitlab/gitaly_client/object_pool_service.rb,
lib/gitlab/gitaly_client/health_check_service.rb,
lib/gitlab/gitaly_client/praefect_info_service.rb,
lib/gitlab/gitaly_client/conflict_files_stitcher.rb

Defined Under Namespace

Modules: AttributesBag, Util Classes: BlobService, BlobsStitcher, Call, CleanupService, CommitService, ConflictFilesStitcher, ConflictsService, Diff, DiffStitcher, HealthCheckService, NamespaceService, ObjectPoolService, OperationService, PraefectInfoService, QueueEnumerator, RefService, RemoteService, RepositoryService, ServerService, StorageSettings, TooManyInvocationsError, WikiFile, WikiPage, WikiService

Constant Summary collapse

PEM_REGEX = 
/\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze
SERVER_VERSION_FILE = 
'GITALY_SERVER_VERSION'
MAXIMUM_GITALY_CALLS = 
30
CLIENT_NAME = 
(Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
GITALY_METADATA_FILENAME = 
'.gitaly-metadata'
MUTEX = 
Mutex.new

Class Method Summary collapse

Class Method Details

.add_call_details(details) ⇒ Object 


356
357
358
359
 
# File 'lib/gitlab/gitaly_client.rb', line 356

def self.add_call_details(details)
  Gitlab::SafeRequestStore['gitaly_call_details'] ||= []
  Gitlab::SafeRequestStore['gitaly_call_details'] << details
end

.add_query_time(duration) ⇒ Object 


185
186
187
188
189
190
 
# File 'lib/gitlab/gitaly_client.rb', line 185

def self.add_query_time(duration)
  return unless Gitlab::SafeRequestStore.active?

  Gitlab::SafeRequestStore[:gitaly_query_time] ||= 0
  Gitlab::SafeRequestStore[:gitaly_query_time] += duration
end

.address(storage) ⇒ Object 


114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/gitlab/gitaly_client.rb', line 114

def self.address(storage)
  params = Gitlab.config.repositories.storages[storage]
  raise "storage not found: #{storage.inspect}" if params.nil?

  address = params['gitaly_address']
  unless address.present?
    raise "storage #{storage.inspect} is missing a gitaly_address"
  end

  unless URI(address).scheme.in?(%w(tcp unix tls))
    raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'"
  end

  address
end

.address_metadata(storage) ⇒ Object 


130
131
132
 
# File 'lib/gitlab/gitaly_client.rb', line 130

def self.address_metadata(storage)
  Base64.strict_encode64(Gitlab::Json.dump(storage => connection_data(storage)))
end

.allow_n_plus_1_calls ⇒ Object 


297
298
299
300
301
302
303
304
305
306
 
# File 'lib/gitlab/gitaly_client.rb', line 297

def self.allow_n_plus_1_calls
  return yield unless Gitlab::SafeRequestStore.active?

  begin
    increment_call_count(:gitaly_call_count_exception_block_depth)
    yield
  ensure
    decrement_call_count(:gitaly_call_count_exception_block_depth)
  end
end

.allow_ref_name_caching ⇒ Object

Normally a FindCommit RPC will cache the commit with its SHA instead of a ref name, since it's possible the branch is mutated afterwards. However, for read-only requests that never mutate the branch, this method allows caching of the ref name directly.


312
313
314
315
316
317
318
319
320
321
322
 
# File 'lib/gitlab/gitaly_client.rb', line 312

def self.allow_ref_name_caching
  return yield unless Gitlab::SafeRequestStore.active?
  return yield if ref_name_caching_allowed?

  begin
    Gitlab::SafeRequestStore[:allow_ref_name_caching] = true
    yield
  ensure
    Gitlab::SafeRequestStore[:allow_ref_name_caching] = false
  end
end

.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block) ⇒ Object

All Gitaly RPC call sites should use GitalyClient.call. This method makes sure that per-request authentication headers are set.

This method optionally takes a block which receives the keyword arguments hash 'kwargs' that will be passed to gRPC. This allows the caller to modify or augment the keyword arguments. The block must return a hash.

For example:

GitalyClient.call(storage, service, rpc, request) do |kwargs|

kwargs.merge(deadline: Time.now + 10)

end

The optional remote_storage keyword argument is used to enable inter-gitaly calls. Say you have an RPC that needs to pull data from one repository to another. For example, to fetch a branch from a (non-deduplicated) fork into the fork parent. In that case you would send an RPC call to the Gitaly server hosting the fork parent, and in the request, you would tell that Gitaly server to pull Git data from the fork. How does that Gitaly server connect to the Gitaly server the forked repo lives on? This is the problem `remote_storage:` solves: it adds address and authentication information to the call, as gRPC metadata (under the `gitaly-servers` header). The request would say “pull from repo X on gitaly-2”. In the Ruby code you pass `remote_storage: 'gitaly-2'`. And then the metadata would say “gitaly-2 is at network address tcp://10.0.1.2:8075”.


166
167
168
 
# File 'lib/gitlab/gitaly_client.rb', line 166

def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout, &block)
  Gitlab::GitalyClient::Call.new(storage, service, rpc, request, remote_storage, timeout).call(&block)
end

.can_use_disk?(storage) ⇒ Boolean

Returns:

  • (Boolean) 

405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
 
# File 'lib/gitlab/gitaly_client.rb', line 405

def self.can_use_disk?(storage)
  cached_value = MUTEX.synchronize do
    @can_use_disk ||= {}
    @can_use_disk[storage]
  end

  return cached_value unless cached_value.nil?

  gitaly_filesystem_id = filesystem_id(storage)
  direct_filesystem_id = filesystem_id_from_disk(storage)

  MUTEX.synchronize do
    @can_use_disk[storage] = gitaly_filesystem_id.present? &&
      gitaly_filesystem_id == direct_filesystem_id
  end
end

.clear_stubs! ⇒ Object 


104
105
106
107
108
 
# File 'lib/gitlab/gitaly_client.rb', line 104

def self.clear_stubs!
  MUTEX.synchronize do
    @stubs = nil
  end
end

.connection_data(storage) ⇒ Object 


134
135
136
 
# File 'lib/gitlab/gitaly_client.rb', line 134

def self.connection_data(storage)
  { 'address' => address(storage), 'token' => token(storage) }
end

.default_timeout ⇒ Object

The default timeout on all Gitaly calls


377
378
379
 
# File 'lib/gitlab/gitaly_client.rb', line 377

def self.default_timeout
  timeout(:gitaly_timeout_default)
end

.enforce_gitaly_request_limits(call_site) ⇒ Object

Ensures that Gitaly is not being abuse through n+1 misuse etc

Raises:


260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
 
# File 'lib/gitlab/gitaly_client.rb', line 260

def self.enforce_gitaly_request_limits(call_site)
  # Only count limits in request-response environments
  return unless Gitlab::SafeRequestStore.active?

  # This is this actual number of times this call was made. Used for information purposes only
  actual_call_count = increment_call_count("gitaly_#{call_site}_actual")

  return unless enforce_gitaly_request_limits?

  # Check if this call is nested within a allow_n_plus_1_calls
  # block and skip check if it is
  return if get_call_count(:gitaly_call_count_exception_block_depth) > 0

  # This is the count of calls outside of a `allow_n_plus_1_calls` block
  # It is used for enforcement but not statistics
  permitted_call_count = increment_call_count("gitaly_#{call_site}_permitted")

  count_stack

  return if permitted_call_count <= MAXIMUM_GITALY_CALLS

  raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks)
end

.execute(storage, service, rpc, request, remote_storage:, timeout:) ⇒ Object 


170
171
172
173
174
175
176
177
178
 
# File 'lib/gitlab/gitaly_client.rb', line 170

def self.execute(storage, service, rpc, request, remote_storage:, timeout:)
  enforce_gitaly_request_limits(:call)
  Gitlab::RequestContext.instance.ensure_deadline_not_exceeded!

  kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage)
  kwargs = yield(kwargs) if block_given?

  stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
end

.expected_server_version ⇒ Object 


367
368
369
370
 
# File 'lib/gitlab/gitaly_client.rb', line 367

def self.expected_server_version
  path = Rails.root.join(SERVER_VERSION_FILE)
  path.read.chomp
end

.fast_timeout ⇒ Object 


381
382
383
 
# File 'lib/gitlab/gitaly_client.rb', line 381

def self.fast_timeout
  timeout(:gitaly_timeout_fast)
end

.filesystem_disk_available(storage) ⇒ Object 


434
435
436
 
# File 'lib/gitlab/gitaly_client.rb', line 434

def self.filesystem_disk_available(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.available
end

.filesystem_disk_used(storage) ⇒ Object 


438
439
440
 
# File 'lib/gitlab/gitaly_client.rb', line 438

def self.filesystem_disk_used(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.used
end

.filesystem_id(storage) ⇒ Object 


422
423
424
 
# File 'lib/gitlab/gitaly_client.rb', line 422

def self.filesystem_id(storage)
  Gitlab::GitalyClient::ServerService.new(storage).storage_info&.filesystem_id
end

.filesystem_id_from_disk(storage) ⇒ Object 


426
427
428
429
430
431
432
 
# File 'lib/gitlab/gitaly_client.rb', line 426

def self.filesystem_id_from_disk(storage)
  metadata_file = File.read(storage_metadata_file_path(storage))
  metadata_hash = Gitlab::Json.parse(metadata_file)
  metadata_hash['gitaly_filesystem_id']
rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError
  nil
end

.get_request_count ⇒ Object

Returns the of the number of Gitaly calls made for this request


345
346
347
 
# File 'lib/gitlab/gitaly_client.rb', line 345

def self.get_request_count
  get_call_count("gitaly_call_actual")
end

.list_call_details ⇒ Object 


361
362
363
364
365
 
# File 'lib/gitlab/gitaly_client.rb', line 361

def self.list_call_details
  return [] unless Gitlab::PerformanceBar.enabled_for_request?

  Gitlab::SafeRequestStore['gitaly_call_details'] || []
end

.long_timeout ⇒ Object 


389
390
391
392
393
394
395
 
# File 'lib/gitlab/gitaly_client.rb', line 389

def self.long_timeout
  if Gitlab::Runtime.web_server?
    default_timeout
  else
    6.hours
  end
end

.medium_timeout ⇒ Object 


385
386
387
 
# File 'lib/gitlab/gitaly_client.rb', line 385

def self.medium_timeout
  timeout(:gitaly_timeout_medium)
end

.query_time ⇒ Object 


180
181
182
183
 
# File 'lib/gitlab/gitaly_client.rb', line 180

def self.query_time
  query_time = Gitlab::SafeRequestStore[:gitaly_query_time] || 0
  query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION)
end

.random_storage ⇒ Object 


110
111
112
 
# File 'lib/gitlab/gitaly_client.rb', line 110

def self.random_storage
  Gitlab.config.repositories.storages.keys.sample
end

.ref_name_caching_allowed? ⇒ Boolean

Returns:

  • (Boolean) 

324
325
326
 
# File 'lib/gitlab/gitaly_client.rb', line 324

def self.ref_name_caching_allowed?
  Gitlab::SafeRequestStore[:allow_ref_name_caching]
end

.request_kwargs(storage, timeout:, remote_storage: nil) ⇒ Object 


212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
 
# File 'lib/gitlab/gitaly_client.rb', line 212

def self.request_kwargs(storage, timeout:, remote_storage: nil)
  metadata = {
    'authorization' => "Bearer #{authorization_token(storage)}",
    'client_name' => CLIENT_NAME
  }

  feature_stack = Thread.current[:gitaly_feature_stack]
  feature = feature_stack && feature_stack[0]
  metadata['call_site'] = feature.to_s if feature
  metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
  metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id
  metadata['gitaly-session-id'] = session_id
  metadata.merge!(Feature::Gitaly.server_feature_flags)

  deadline_info = request_deadline(timeout)
  metadata.merge!(deadline_info.slice(:deadline_type))

  { metadata: metadata, deadline: deadline_info[:deadline] }
end

.reset_counts ⇒ Object 


349
350
351
352
353
354
 
# File 'lib/gitlab/gitaly_client.rb', line 349

def self.reset_counts
  return unless Gitlab::SafeRequestStore.active?

  Gitlab::SafeRequestStore["gitaly_call_actual"] = 0
  Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0
end

.session_id ⇒ Object 


248
249
250
 
# File 'lib/gitlab/gitaly_client.rb', line 248

def self.session_id
  Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid
end

.storage_metadata_file_path(storage) ⇒ Object 


397
398
399
400
401
402
403
 
# File 'lib/gitlab/gitaly_client.rb', line 397

def self.storage_metadata_file_path(storage)
  Gitlab::GitalyClient::StorageSettings.allow_disk_access do
    File.join(
      Gitlab.config.repositories.storages[storage].legacy_disk_path, GITALY_METADATA_FILENAME
    )
  end
end

.stub(name, storage) ⇒ Object 


35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/gitlab/gitaly_client.rb', line 35

def self.stub(name, storage)
  MUTEX.synchronize do
    @stubs ||= {}
    @stubs[storage] ||= {}
    @stubs[storage][name] ||= begin
      klass = stub_class(name)
      addr = stub_address(storage)
      creds = stub_creds(storage)
      klass.new(addr, creds, interceptors: interceptors, channel_args: channel_args)
    end
  end
end

.stub_address(storage) ⇒ Object 


100
101
102
 
# File 'lib/gitlab/gitaly_client.rb', line 100

def self.stub_address(storage)
  address(storage).sub(%r{^tcp://|^tls://}, '')
end

.stub_cert_paths ⇒ Object 


65
66
67
68
69
 
# File 'lib/gitlab/gitaly_client.rb', line 65

def self.stub_cert_paths
  cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"]
  cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE
  cert_paths
end

.stub_certs ⇒ Object 


71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/gitlab/gitaly_client.rb', line 71

def self.stub_certs
  return @certs if @certs

  @certs = stub_cert_paths.flat_map do |cert_file|
    File.read(cert_file).scan(PEM_REGEX).map do |cert|
      OpenSSL::X509::Certificate.new(cert).to_pem
    rescue OpenSSL::OpenSSLError => e
      Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, cert_file: cert_file)
      nil
    end.compact
  end.uniq.join("\n")
end

.stub_class(name) ⇒ Object 


92
93
94
95
96
97
98
 
# File 'lib/gitlab/gitaly_client.rb', line 92

def self.stub_class(name)
  if name == :health_check
    Grpc::Health::V1::Health::Stub
  else
    Gitaly.const_get(name.to_s.camelcase.to_sym, false).const_get(:Stub, false)
  end
end

.stub_creds(storage) ⇒ Object 


84
85
86
87
88
89
90
 
# File 'lib/gitlab/gitaly_client.rb', line 84

def self.stub_creds(storage)
  if URI(address(storage)).scheme == 'tls'
    GRPC::Core::ChannelCredentials.new stub_certs
  else
    :this_channel_is_insecure
  end
end

.timestamp(time) ⇒ Object 


372
373
374
 
# File 'lib/gitlab/gitaly_client.rb', line 372

def self.timestamp(time)
  Google::Protobuf::Timestamp.new(seconds: time.to_i)
end

.token(storage) ⇒ Object 


252
253
254
255
256
257
 
# File 'lib/gitlab/gitaly_client.rb', line 252

def self.token(storage)
  params = Gitlab.config.repositories.storages[storage]
  raise "storage not found: #{storage.inspect}" if params.nil?

  params['gitaly_token'].presence || Gitlab.config.gitaly['token']
end