Class: Datadog::Profiling::Collectors::Stack

Inherits:
Worker
  • Object
show all
Includes:
Workers::Polling
Defined in:
lib/ddtrace/profiling/collectors/stack.rb

Overview

Collects stack trace samples from Ruby threads for both CPU-time (if available) and wall-clock. Runs on its own background thread.

Constant Summary collapse

DEFAULT_MAX_TIME_USAGE_PCT = 
2.0
MIN_INTERVAL = 
0.01
THREAD_LAST_CPU_TIME_KEY = 
:datadog_profiler_last_cpu_time

Constants included from Workers::Polling

Workers::Polling::SHUTDOWN_TIMEOUT

Instance Attribute Summary collapse

Attributes inherited from Worker

#task

Instance Method Summary collapse

Methods included from Workers::Polling

#enabled=, #enabled?, included, #stop

Constructor Details

#initialize(recorder, max_frames:, trace_identifiers_helper:, ignore_thread: nil, max_time_usage_pct: DEFAULT_MAX_TIME_USAGE_PCT, thread_api: Thread, fork_policy: Workers::Async::Thread::FORK_POLICY_RESTART, interval: MIN_INTERVAL, enabled: true) ⇒ Stack

Returns a new instance of Stack.



30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 30

def initialize(
  recorder,
  max_frames:,
  trace_identifiers_helper:, # Usually an instance of Datadog::Profiling::TraceIdentifiers::Helper
  ignore_thread: nil,
  max_time_usage_pct: DEFAULT_MAX_TIME_USAGE_PCT,
  thread_api: Thread,
  fork_policy: Workers::Async::Thread::FORK_POLICY_RESTART, # Restart in forks by default
  interval: MIN_INTERVAL,
  enabled: true
)
  @recorder = recorder
  @max_frames = max_frames
  @trace_identifiers_helper = trace_identifiers_helper
  @ignore_thread = ignore_thread
  @max_time_usage_pct = max_time_usage_pct
  @thread_api = thread_api

  # Workers::Async::Thread settings
  self.fork_policy = fork_policy

  # Workers::IntervalLoop settings
  self.loop_base_interval = interval

  # Workers::Polling settings
  self.enabled = enabled

  @warn_about_missing_cpu_time_instrumentation_only_once = Datadog::Utils::OnlyOnce.new

  # Cache this proc, since it's pretty expensive to keep recreating it
  @build_backtrace_location = method(:build_backtrace_location).to_proc
  # Cache this buffer, since it's pretty expensive to keep accessing it
  @stack_sample_event_recorder = recorder[Events::StackSample]
end

Instance Attribute Details

#ignore_thread ⇒ Object (readonly)

Returns the value of attribute ignore_thread.



22
23
24
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 22

def ignore_thread
  @ignore_thread
end

#max_frames ⇒ Object (readonly)

Returns the value of attribute max_frames.



22
23
24
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 22

def max_frames
  @max_frames
end

#max_time_usage_pct ⇒ Object (readonly)

Returns the value of attribute max_time_usage_pct.



22
23
24
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 22

def max_time_usage_pct
  @max_time_usage_pct
end

#recorder ⇒ Object (readonly)

Returns the value of attribute recorder.



22
23
24
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 22

def recorder
  @recorder
end

#thread_api ⇒ Object (readonly)

Returns the value of attribute thread_api.



22
23
24
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 22

def thread_api
  @thread_api
end

#trace_identifiers_helper ⇒ Object (readonly)

Returns the value of attribute trace_identifiers_helper.



22
23
24
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 22

def trace_identifiers_helper
  @trace_identifiers_helper
end

Instance Method Details

#build_backtrace_location(_id, base_label, lineno, path) ⇒ Object 



200
201
202
203
204
205
206
207
208
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 200

def build_backtrace_location(_id, base_label, lineno, path)
  string_table = @stack_sample_event_recorder.string_table

  Profiling::BacktraceLocation.new(
    string_table.fetch_string(base_label),
    lineno,
    string_table.fetch_string(path)
  )
end

#collect_and_wait ⇒ Object 



79
80
81
82
83
84
85
86
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 79

def collect_and_wait
  run_time = Datadog::Utils::Time.measure do
    collect_events
  end

  # Update wait time to throttle profiling
  self.loop_wait_time = compute_wait_time(run_time)
end

#collect_events ⇒ Object 



88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 88

def collect_events
  events = []

  # Compute wall time interval
  current_wall_time = Datadog::Utils::Time.get_time
  last_wall_time = if instance_variable_defined?(:@last_wall_time)
                     @last_wall_time
                   else
                     current_wall_time
                   end

  wall_time_interval_ns = ((current_wall_time - last_wall_time).round(9) * 1e9).to_i
  @last_wall_time = current_wall_time

  # Collect backtraces from each thread
  thread_api.list.each do |thread|
    next unless thread.alive?
    next if ignore_thread.is_a?(Proc) && ignore_thread.call(thread)

    event = collect_thread_event(thread, wall_time_interval_ns)
    events << event unless event.nil?
  end

  # Send events to recorder
  recorder.push(events) unless events.empty?

  events
end

#collect_thread_event(thread, wall_time_interval_ns) ⇒ Object 



117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 117

def collect_thread_event(thread, wall_time_interval_ns)
  locations = thread.backtrace_locations
  return if locations.nil?

  # Get actual stack size then trim the stack
  stack_size = locations.length
  locations = locations[0..(max_frames - 1)]

  # Convert backtrace locations into structs
  locations = convert_backtrace_locations(locations)

  thread_id = thread.respond_to?(:pthread_thread_id) ? thread.pthread_thread_id : thread.object_id
  trace_id, span_id, trace_resource_container = trace_identifiers_helper.trace_identifiers_for(thread)
  cpu_time = get_cpu_time_interval!(thread)

  Events::StackSample.new(
    nil,
    locations,
    stack_size,
    thread_id,
    trace_id,
    span_id,
    trace_resource_container,
    cpu_time,
    wall_time_interval_ns
  )
end

#compute_wait_time(used_time) ⇒ Object 



169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 169

def compute_wait_time(used_time)
  # We took used_time to get the last sample.
  #
  # What we're computing here is -- if used_time corresponds to max_time_usage_pct of the time we should
  # spend working, how much is (100% - max_time_usage_pct) of the time?
  #
  # For instance, if we took 10ms to sample, and max_time_usage_pct is 1%, then the other 99% is 990ms, which
  # means we need to sleep for 990ms to guarantee that we don't spend more than 1% of the time working.
  used_time_ns = used_time * 1e9
  interval = (used_time_ns / (max_time_usage_pct / 100.0)) - used_time_ns
  [interval / 1e9, MIN_INTERVAL].max
end

#convert_backtrace_locations(locations) ⇒ Object

Convert backtrace locations into structs Re-use old backtrace location objects if they already exist in the buffer



184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 184

def convert_backtrace_locations(locations)
  locations.collect do |location|
    # Re-use existing BacktraceLocation if identical copy, otherwise build a new one.
    @stack_sample_event_recorder.cache(:backtrace_locations).fetch(
      # Function name
      location.base_label,
      # Line number
      location.lineno,
      # Filename
      location.path,
      # Build function
      &@build_backtrace_location
    )
  end
end

#get_cpu_time_interval!(thread) ⇒ Object 



145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 145

def get_cpu_time_interval!(thread)
  # Return if we can't get the current CPU time
  unless thread.respond_to?(:cpu_time_instrumentation_installed?) && thread.cpu_time_instrumentation_installed?
    warn_about_missing_cpu_time_instrumentation(thread)
    return
  end

  current_cpu_time_ns = thread.cpu_time(:nanosecond)

  # NOTE: This can still be nil even when all of the checks above passed because of a race: there's a bit of
  # initialization that needs to be done by the thread itself, and it's possible for us to try to sample
  # *before* the thread had time to finish the initialization
  return unless current_cpu_time_ns

  last_cpu_time_ns = (thread.thread_variable_get(THREAD_LAST_CPU_TIME_KEY) || current_cpu_time_ns)
  interval = current_cpu_time_ns - last_cpu_time_ns

  # Update CPU time for thread
  thread.thread_variable_set(THREAD_LAST_CPU_TIME_KEY, current_cpu_time_ns)

  # Return interval
  interval
end

#loop_back_off? ⇒ Boolean

Returns:

  • (Boolean) 


75
76
77
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 75

def loop_back_off?
  false
end

#perform ⇒ Object 



71
72
73
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 71

def perform
  collect_and_wait
end

#start ⇒ Object 



65
66
67
68
69
 
# File 'lib/ddtrace/profiling/collectors/stack.rb', line 65

def start
  @last_wall_time = Datadog::Utils::Time.get_time
  reset_cpu_time_tracking
  perform
end