Class: Elasticity::SparkStep

Inherits:
Object
  • Object
show all
Includes:
JobFlowStep
Defined in:
lib/elasticity/spark_step.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from JobFlowStep

#aws_installation_step_name, #aws_installation_steps, included, #requires_installation?, steps_requiring_installation

Constructor Details

#initialize(jar, main_class) ⇒ SparkStep

Returns a new instance of SparkStep.



14
15
16
17
18
19
20
21
 
# File 'lib/elasticity/spark_step.rb', line 14

def initialize(jar, main_class)
  @name = 'Elasticity Spark Step'
  @main_class = main_class
  @jar = jar
  @spark_arguments = {}
  @app_arguments = {}
  @action_on_failure = 'TERMINATE_JOB_FLOW'
end

Instance Attribute Details

#action_on_failure ⇒ Object

Returns the value of attribute action_on_failure.



12
13
14
 
# File 'lib/elasticity/spark_step.rb', line 12

def action_on_failure
  @action_on_failure
end

#app_arguments ⇒ Object

Returns the value of attribute app_arguments.



11
12
13
 
# File 'lib/elasticity/spark_step.rb', line 11

def app_arguments
  @app_arguments
end

#jar ⇒ Object

Returns the value of attribute jar.



9
10
11
 
# File 'lib/elasticity/spark_step.rb', line 9

def jar
  @jar
end

#main_class ⇒ Object

Returns the value of attribute main_class.



8
9
10
 
# File 'lib/elasticity/spark_step.rb', line 8

def main_class
  @main_class
end

#name ⇒ Object

Returns the value of attribute name.



7
8
9
 
# File 'lib/elasticity/spark_step.rb', line 7

def name
  @name
end

#spark_arguments ⇒ Object

Returns the value of attribute spark_arguments.



10
11
12
 
# File 'lib/elasticity/spark_step.rb', line 10

def spark_arguments
  @spark_arguments
end

Instance Method Details

#to_aws_step(_) ⇒ Object 



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/elasticity/spark_step.rb', line 23

def to_aws_step(_)
  args = %W(spark-submit --class #{@main_class})
  spark_arguments.each do |arg, value|
    args << "--#{arg}" << value
  end
  args.push(@jar)
  app_arguments.each do |arg, value|
    args << "--#{arg}" << value
  end
  {
    :name => @name,
    :action_on_failure => @action_on_failure,
    :hadoop_jar_step => {
      :jar => 'command-runner.jar',
      :args => args
    }
  }
end