Module: HadoopMetrics2::API

Included in:
DataNode, NameNode, NodeManager, ResourceManager
Defined in:
lib/hadoop_metrics2/api.rb

Constant Summary collapse

GCNameMap = 
{
  'PS Scavenge' => 'minor',  # for backward compatibility
  'PS MarkSweep' => 'major', # for backward compatibility
  'ConcurrentMarkSweep' => 'c_mark_sweep',
  'ParNew' => 'par_new'
}
MegaByte = 
1024.0 * 1024

Instance Attribute Summary collapse

Instance Method Summary collapse

Dynamic Method Handling

This class handles dynamic methods through the method_missing method

#method_missing(method, *args) ⇒ Object (private) 



146
147
148
149
 
# File 'lib/hadoop_metrics2/api.rb', line 146

def method_missing(method, *args)
  category, target = method.to_s.split('_', 2)
  group_by(category, target, *args)
end

Instance Attribute Details

#name ⇒ Object (readonly)

Returns the value of attribute name.



17
18
19
 
# File 'lib/hadoop_metrics2/api.rb', line 17

def name
  @name
end

Instance Method Details

#gc ⇒ Object 



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/hadoop_metrics2/api.rb', line 44

def gc
  disable_snake_case {
    result = query_jmx('java.lang:type=GarbageCollector,name=*').map { |jmx_gc_info|
      return nil if jmx_gc_info['LastGcInfo'].nil?

      gc_info = {'type' => GCNameMap[jmx_gc_info['Name']]}
      gc_info['estimated_time'] = jmx_gc_info['CollectionTime']
      gc_info['count'] = jmx_gc_info['CollectionCount']

      last_gc_info = jmx_gc_info['LastGcInfo']
      gc_info['last_start'] = last_gc_info['startTime']
      gc_info['last_duration'] = last_gc_info['duration']
      gc_info['after_gc'] = calc_memory_usage(last_gc_info)

      gc_info
    }
  }
end

#get_jmx(query, json_fields = []) ⇒ Object 



87
88
89
 
# File 'lib/hadoop_metrics2/api.rb', line 87

def get_jmx(query, json_fields = [])
  via_jmx('get', query, json_fields)
end

#initialize(host, port, master, opts = {}) ⇒ Object 



7
8
9
10
11
12
13
14
15
 
# File 'lib/hadoop_metrics2/api.rb', line 7

def initialize(host, port, master, opts = {})
  @endpoint = "#{host}:#{port}"
  @metrics_endpoint = master ?
    URI("http://#{@endpoint}/ws/v1/cluster/metrics") : URI("http://#{@endpoint}/ws/v1/node/info")
  snake_case = opts.has_key?(:snake_case) ? opts[:snake_case] : true
  @name = opts[:name] || host
  @metrics_cache = nil
  @scheduler_cache = nil
end

#memory ⇒ Object 



65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/hadoop_metrics2/api.rb', line 65

def memory
  disable_snake_case {
    result = {}

    memory = query_jmx('java.lang:type=Memory').first
    heap, non_heap = memory['HeapMemoryUsage'], memory['NonHeapMemoryUsage']
    result['committed'] = (heap['committed'] + non_heap['committed']) / MegaByte
    result['used'] = (heap['used'] + non_heap['used']) / MegaByte
    result['max'] = (heap['max'] + non_heap['max']) / MegaByte

    # Can we use 'max' attribute instead of -Xmx option?
    arguments = get_jmx('java.lang:type=Runtime::InputArguments').first['InputArguments']
    result['mx_option'] = arguments.select { |arg| arg =~ /-Xmx(.*)m/ }.last["-Xmx".size..-2].to_i

    result
  }
end

#metrics(force = true) ⇒ Object 



19
20
21
22
23
24
25
26
 
# File 'lib/hadoop_metrics2/api.rb', line 19

def metrics(force = true)
  if !@metrics_cache.nil? and !force
    return @metrics_cache
  end

  @metrics_cache = HadoopMetrics2.get_response(@metrics_endpoint)
  @metrics_cache
end

#query_jmx(query, json_fields = []) ⇒ Object 



83
84
85
 
# File 'lib/hadoop_metrics2/api.rb', line 83

def query_jmx(query, json_fields = [])
  via_jmx('qry', query, json_fields)
end

#scheduler(force = true) ⇒ Object 



28
29
30
31
32
33
34
35
 
# File 'lib/hadoop_metrics2/api.rb', line 28

def scheduler(force = true)
  if !@scheduler_cache.nil? and !force
    return @scheduler_cache
  end

  @scheduler_cache = HadoopMetrics2.get_response(URI("http://#{@endpoint}/ws/v1/cluster/scheduler"))
  @scheduler_cache
end

#via_jmx(type, query, json_fields = []) ⇒ Object 



91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/hadoop_metrics2/api.rb', line 91

def via_jmx(type, query, json_fields = [])
  HadoopMetrics2.get_response(URI("http://#{@endpoint}/jmx?#{type}=#{query}"))['beans'].map { |jmx_json|
    json_fields.each { |f|
      jmx_json[f] = JSON.parse(jmx_json[f])
    }
    if @snake_case
      jmx_json = HadoopMetrics2.snake_cased(jmx_json)
    end

    jmx_json
  }
end