Class: SparkToolkit::YARN::Client

Inherits:
Object
  • Object
show all
Defined in:
lib/spark_toolkit/hadoop/yarn/client.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(conf = nil) ⇒ Client

Returns a new instance of Client.



8
9
10
11
12
 
# File 'lib/spark_toolkit/hadoop/yarn/client.rb', line 8

def initialize(conf=nil)
  initalise
  @conf = conf
  init conf if conf
end

Instance Attribute Details

#conf ⇒ Object (readonly)

Returns the value of attribute conf.



6
7
8
 
# File 'lib/spark_toolkit/hadoop/yarn/client.rb', line 6

def conf
  @conf
end

Instance Method Details

#get_application_attempts(app_id) ⇒ Object

get_container_report(container_id)



24
25
26
 
# File 'lib/spark_toolkit/hadoop/yarn/client.rb', line 24

def get_application_attempts(app_id)
  getApplicationAttempts(app_id).to_a
end

#get_application_logs(appid, dev = :all) ⇒ Object

Available devs are:

  • :all

  • :stdout

  • :stderr



53
54
55
56
57
 
# File 'lib/spark_toolkit/hadoop/yarn/client.rb', line 53

def get_application_logs(appid, dev=:all)
  @conf ||= SparkToolkit::Conf::Configuration.new
  @log_accssor ||= SparkToolkit::YARN::LogAccessor.new(@conf)
  @log_accssor.get_logs(appid, dev)
end

#get_applications ⇒ Object 



14
15
16
 
# File 'lib/spark_toolkit/hadoop/yarn/client.rb', line 14

def get_applications
  getApplications.to_a
end

#get_cluster_report ⇒ Object 



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
 
# File 'lib/spark_toolkit/hadoop/yarn/client.rb', line 33

def get_cluster_report
  sum = get_node_reports.reduce([0,0,0,0]) do |sum, report|
    sum[0] += report.get_total_memory
    sum[1] += report.get_used_memory
    sum[2] += report.get_total_vcores
    sum[3] += report.get_used_vcores
    sum
  end
  {
    total_memory: sum[0],
    used_memory: sum[1],
    total_vcores: sum[2],
    used_vcores: sum[3],
  }
end

#get_containers(app_id) ⇒ Object

get_application_report(app_id)



19
20
21
 
# File 'lib/spark_toolkit/hadoop/yarn/client.rb', line 19

def get_containers(app_id)
  getContainers(app_id).to_a
end

#get_node_reports ⇒ Object

get_attempt_report(app_id)



29
30
31
 
# File 'lib/spark_toolkit/hadoop/yarn/client.rb', line 29

def get_node_reports
  getNodeReports.to_a
end

#initalise ⇒ Object 



7
 
# File 'lib/spark_toolkit/hadoop/yarn/client.rb', line 7

alias_method :initalise, :initialize