Class: Elasticity::StreamingStep

Inherits:
Object
  • Object
show all
Includes:
JobFlowStep
Defined in:
lib/elasticity/streaming_step.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from JobFlowStep

#aws_installation_step_name, #aws_installation_steps, included, #requires_installation?, steps_requiring_installation

Constructor Details

#initialize(input_bucket, output_bucket, mapper, reducer, *arguments) ⇒ StreamingStep



15
16
17
18
19
20
21
22
23
# File 'lib/elasticity/streaming_step.rb', line 15

def initialize(input_bucket, output_bucket, mapper, reducer, *arguments)
  @name = 'Elasticity Streaming Step'
  @action_on_failure = 'TERMINATE_JOB_FLOW'
  @input_bucket = input_bucket
  @output_bucket = output_bucket
  @mapper = mapper
  @reducer = reducer
  @arguments = arguments || []
end

Instance Attribute Details

#action_on_failureObject

Returns the value of attribute action_on_failure



8
9
10
# File 'lib/elasticity/streaming_step.rb', line 8

def action_on_failure
  @action_on_failure
end

#argumentsObject

Returns the value of attribute arguments



13
14
15
# File 'lib/elasticity/streaming_step.rb', line 13

def arguments
  @arguments
end

#input_bucketObject

Returns the value of attribute input_bucket



9
10
11
# File 'lib/elasticity/streaming_step.rb', line 9

def input_bucket
  @input_bucket
end

#mapperObject

Returns the value of attribute mapper



11
12
13
# File 'lib/elasticity/streaming_step.rb', line 11

def mapper
  @mapper
end

#nameObject

Returns the value of attribute name



7
8
9
# File 'lib/elasticity/streaming_step.rb', line 7

def name
  @name
end

#output_bucketObject

Returns the value of attribute output_bucket



10
11
12
# File 'lib/elasticity/streaming_step.rb', line 10

def output_bucket
  @output_bucket
end

#reducerObject

Returns the value of attribute reducer



12
13
14
# File 'lib/elasticity/streaming_step.rb', line 12

def reducer
  @reducer
end

Instance Method Details

#to_aws_step(job_flow) ⇒ Object



25
26
27
28
29
30
31
# File 'lib/elasticity/streaming_step.rb', line 25

def to_aws_step(job_flow)
  step = Elasticity::CustomJarStep.new('/home/hadoop/contrib/streaming/hadoop-streaming.jar')
  step.name = @name
  step.action_on_failure = @action_on_failure
  step.arguments = ['-input', @input_bucket, '-output', @output_bucket, '-mapper', @mapper, '-reducer', @reducer] + @arguments
  step.to_aws_step(job_flow)
end