Module: Buildkite::TestCollector::OTel

Defined in:
lib/buildkite/test_collector/otel.rb,
lib/buildkite/test_collector/otel/span_filter.rb,
lib/buildkite/test_collector/otel/child_span_forwarder.rb,
lib/buildkite/test_collector/otel/span_metrics_reporter.rb,
lib/buildkite/test_collector/otel/test_span_metrics_reporter.rb,
lib/buildkite/test_collector/otel/child_span_metrics_reporter.rb

Constant Summary collapse

DEFAULT_ENDPOINT = 
"https://tests-otlp.buildkite.com/v1/traces"
RUN_KEY_FORMAT =

Accepted by the Buildkite OTLP traces receiver for its run-key header (Analytics::API::TracesController::RUN_KEY_FORMAT); keep them in sync.

/\A[!-~]{1,255}\z/
EXECUTION_VIA_ATTRIBUTE = 
"buildkite.execution.via"
RESULT_ATTRIBUTE = 
"test.case.result.status"
TAG_ATTRIBUTE_PREFIX = 
"buildkite.tag."
RESULT_STATUSES =

OpenTelemetry has no standard value for skipped tests.

{
  "passed" => "pass",
  "failed" => "fail",
  "skipped" => "skipped",
}.freeze
PROCESSOR_TIMEOUT_SECONDS = 
30
TRACER_NAME = 
"buildkite-test-collector"
TEST_SPAN_NAME = 
"test.execution"
EXPORT_TIMEOUT_MILLISECONDS =

Passed explicitly to both batch processors so the process-wide OTEL_BSP_* settings do not affect Buildkite export. SDK defaults unless noted.

30_000
TEST_SPAN_MAX_QUEUE_SIZE = 
8_192
TEST_SPAN_MAX_EXPORT_BATCH_SIZE =

The server rejects gzip request bodies over 900 KiB. Failures at the message and stack trace limits with high-entropy text gzip to about 4.2 KiB per span, so 120 leaves headroom; typical batches are ~0.3 KiB per span.

120
TEST_SPAN_SCHEDULE_DELAY_MILLISECONDS = 
1_000
CHILD_SPAN_MAX_QUEUE_SIZE = 
2_048
CHILD_SPAN_MAX_EXPORT_BATCH_SIZE = 
512
CHILD_SPAN_SCHEDULE_DELAY_MILLISECONDS = 
5_000
SPAN_PROCESSOR_SIZE_LIMIT =

Sanity cap for batch and queue overrides.

2**31 - 1
TEST_SPAN_ATTRIBUTE_LENGTH_LIMIT =

Keep in sync with Test.NAME_MAX_LENGTH (10,240) in ta-ingestion. The SDK truncates to 10,240 characters plus "..." and the server truncates that back to 10,240 before deriving the test ID, so a long test name keeps the same identity as the JSON upload.

10_243
TEST_SPAN_EVENT_ATTRIBUTE_LENGTH_LIMIT = 
16_384
TEST_SPAN_EVENT_COUNT_LIMIT = 
100
FAILURE_REASON_MAX_LENGTH =

Mirrors TestResult.FAILURE_REASON_MAX_LENGTH in ta-ingestion.

1_024
EXCEPTION_MESSAGE_MAX_LENGTH =

Keep in sync with TestResult.FAILURE_EXPANDED_MAX_LENGTH (10,240) in ta-ingestion, which keeps that many message characters per execution and truncates away the trailing "...". Stack traces have a separate 100 KiB quota, so they keep the event attribute limit.

10_243

Class Method Summary collapse

Class Method Details

.annotate(content) ⇒ Object

Records a point-in-time annotation as an event on whichever span is current, which during a test is the test's own trace. Safe to call when export is off or nothing is recording: it just does nothing.



263
264
265
266
267
268
269
270
271
272
273
 
# File 'lib/buildkite/test_collector/otel.rb', line 263

def annotate(content)
  return unless enabled?

  span = OpenTelemetry::Trace.current_span
  return unless span.recording?

  span.add_event("test.annotation", attributes: { "buildkite.annotation" => content.to_s })
rescue Exception => e # rubocop:disable Lint/RescueException
  ExceptionHandling.reraise_fatal(e)
  warn "[buildkite-test_collector] Could not annotate OpenTelemetry test span: #{e.class}: #{e.message}"
end

.api_token ⇒ Object 



146
147
148
149
 
# File 'lib/buildkite/test_collector/otel.rb', line 146

def api_token
  token = ENV["BUILDKITE_TESTS_OTLP_TOKEN"]&.strip
  token unless token.nil? || token.empty?
end

.configure!(endpoint: self.endpoint, api_token: self.api_token, run_env: {}, span_filter: nil, tags: {}) ⇒ Object 



151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/buildkite/test_collector/otel.rb', line 151

def configure!(endpoint: self.endpoint, api_token: self.api_token, run_env: {}, span_filter: nil, tags: {})
  run_key = run_env["key"]
  # The receiver rejects every batch sent with an invalid run key, so
  # fail before loading anything; an enabled process already passed this.
  if !enabled? && !valid_run_key?(run_key)
    warn_invalid_run_key(run_key)
    return
  end

  if enabled?
    # One process serves one run: the exporters and providers live for
    # the whole process, so run identity is fixed at first configure.
    # Only credentials may change within that lifetime - a warm worker
    # re-running a suite can bring a fresh (e.g. expiring OIDC) token
    # that the exporters' snapshotted Authorization headers would
    # otherwise never learn about. A different run key means a new run,
    # which needs a new process; warn rather than misattribute silently.
    warn_run_mismatch(run_env)
    refresh_authorization(api_token)
    return
  end

  require "opentelemetry/sdk"
  require "opentelemetry/exporter/otlp"
  require "opentelemetry/trace/propagation/trace_context"

  exempt_from_vcr(endpoint)
  exempt_from_webmock(endpoint)

  @api_token = api_token
  @run_key = run_env["key"]
  headers = request_headers(run_env, api_token)

  # Resources identify the entities that produced the telemetry. Details
  # about the Test Engine run and test framework describe each execution
  # instead, so keep them on the test span rather than every child span.
  resource = producer_resource(run_env)
  @run_attributes = run_attributes(run_env, tags)

  @test_span_provider = build_test_span_provider(endpoint, headers, resource)
  @tracer = @test_span_provider.tracer(TRACER_NAME, Buildkite::TestCollector::VERSION)
  configure_child_export(endpoint, headers, resource, span_filter: span_filter)
  register_shutdown_at_exit
rescue Exception => e # rubocop:disable Lint/RescueException
  ExceptionHandling.reraise_fatal(e)
  warn "[buildkite-test_collector] OpenTelemetry span export disabled: #{e.class}: #{e.message}"
  shutdown
end

.current_timestamp ⇒ Object

"Now" as the SDK would stamp it: the realtime clock, in seconds. Not Time.now, which suites that freeze time (Timecop) fake out.



243
244
245
 
# File 'lib/buildkite/test_collector/otel.rb', line 243

def current_timestamp
  Rational(Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond), 1_000_000_000)
end

.enabled? ⇒ Boolean

Returns:

  • (Boolean) 


137
138
139
 
# File 'lib/buildkite/test_collector/otel.rb', line 137

def enabled?
  !@tracer.nil?
end

.endpoint ⇒ Object

bktec's relay sets its loopback listener here and its credential in api_token.



142
143
144
 
# File 'lib/buildkite/test_collector/otel.rb', line 142

def endpoint
  ENV["BUILDKITE_ANALYTICS_OTLP_ENDPOINT"] || DEFAULT_ENDPOINT
end

.finish_test_span(span, test:, end_timestamp: nil) ⇒ Object

Each step warns and moves on rather than raising, so the span always finishes: a half-described execution beats a missing one.



249
250
251
252
253
254
255
256
257
258
 
# File 'lib/buildkite/test_collector/otel.rb', line 249

def finish_test_span(span, test:, end_timestamp: nil)
  return unless span

  record_result(span, test)
  describe_test(span, test)
  finish_span(span, end_timestamp)
rescue Exception => e # rubocop:disable Lint/RescueException
  ExceptionHandling.reraise_fatal(e)
  warn "[buildkite-test_collector] Could not finish OpenTelemetry test span: #{e.class}: #{e.message}"
end

.force_flush ⇒ Object

Pushes any finished spans out now without stopping export. Used at the end of a suite when the process (and maybe another suite run) lives on.



277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
 
# File 'lib/buildkite/test_collector/otel.rb', line 277

def force_flush
  error = each_export_queue(PROCESSOR_TIMEOUT_SECONDS) do |queue, remaining|
    queue.force_flush(timeout: remaining)
  end
  if error
    warn "[buildkite-test_collector] Could not flush OpenTelemetry spans: #{error.class}: #{error.message}"
  end
  # Report what this suite run has dropped so far. The SDK's flush stops
  # at the first rejected batch and re-queues the rest, so a persistent
  # failure leaves a balance that drains, and is reported, at shutdown.
  warn_dropped_totals
rescue Exception => e # rubocop:disable Lint/RescueException
  ExceptionHandling.reraise_fatal(e)
  warn "[buildkite-test_collector] Could not flush OpenTelemetry spans: #{e.class}: #{e.message}"
end

.shutdown ⇒ Object 



293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
 
# File 'lib/buildkite/test_collector/otel.rb', line 293

def shutdown
  forwarder_error = deactivate_child_span_forwarder(@child_span_forwarder)
  export_error = each_export_queue(PROCESSOR_TIMEOUT_SECONDS) do |queue, remaining|
    queue.shutdown(timeout: remaining)
  end
  error = forwarder_error || export_error
  if error
    warn "[buildkite-test_collector] Could not shut down OpenTelemetry span export: #{error.class}: #{error.message}"
  end
  warn_dropped_totals
rescue Exception => e # rubocop:disable Lint/RescueException
  ExceptionHandling.reraise_fatal(e)
  warn "[buildkite-test_collector] Could not shut down OpenTelemetry span export: #{e.class}: #{e.message}"
ensure
  @test_span_provider = nil
  @child_span_processor = nil
  @child_span_forwarder = nil
  @exporters = nil
  ExporterGuard.reset
  @test_span_metrics_reporter = nil
  @child_span_metrics_reporter = nil
  @api_token = nil
  @run_attributes = nil
  @run_key = nil
  @tracer = nil
end

.start_test_span(test:) ⇒ Object 



200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
 
# File 'lib/buildkite/test_collector/otel.rb', line 200

def start_test_span(test:)
  return unless enabled?

  # The SDK retains the earliest attributes at its configured limit.
  # These three are the minimum needed to synthesize an execution. The
  # span is the submission, so every test span must carry the via marker.
  attributes = { EXECUTION_VIA_ATTRIBUTE => "otlp" }
  run_key = (@run_attributes || {})["buildkite.run_key"]
  attributes["buildkite.run_key"] = run_key if run_key
  # Reserve the result's position before test code can consume the
  # SDK's attribute budget. finish_test_span replaces this value.
  attributes[RESULT_ATTRIBUTE] = "unset"
  test.otel_attributes.each do |key, value|
    next if value.nil? || attributes.key?(key) || key.start_with?(TAG_ATTRIBUTE_PREFIX)

    attributes[key] = value
  end

  @tracer.start_span(
    TEST_SPAN_NAME,
    with_parent: OpenTelemetry::Context.empty,
    attributes: attributes,
    links: job_span_links,
    kind: :internal,
  )
rescue Exception => e # rubocop:disable Lint/RescueException
  ExceptionHandling.reraise_fatal(e)
  # The example still runs, but with no span it reaches neither upload
  # path, so report it as a missing result rather than a stray warning.
  @test_span_metrics_reporter&.record_start_failure(e)
  nil
end

.with_test_span(span) ⇒ Object 



233
234
235
236
237
238
239
 
# File 'lib/buildkite/test_collector/otel.rb', line 233

def with_test_span(span)
  return yield unless span

  OpenTelemetry::Context.with_value(test_span_context_key, span.context.trace_id) do
    OpenTelemetry::Trace.with_span(span) { yield }
  end
end