Class: Elasticrawl::Cluster

Inherits:
Object
  • Object
show all
Defined in:
lib/elasticrawl/cluster.rb

Overview

Configures the cluster settings for the job flow that will be launched. These settings are loaded from ~/.elasticrawl/cluster.yml.

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Cluster

Returns a new instance of Cluster.



5
6
7
8
9
 
# File 'lib/elasticrawl/cluster.rb', line 5

def initialize
  @master_group = instance_group('master')
  @core_group = instance_group('core')
  @task_group = instance_group('task') if has_task_group?
end

Instance Method Details

#cluster_desc ⇒ Object

Describes the instances that will be launched. This is used by the job confirmation messages.



28
29
30
31
32
33
34
35
 
# File 'lib/elasticrawl/cluster.rb', line 28

def cluster_desc
  cluster_desc = <<-HERE
Cluster configuration
Master: #{instance_group_desc(@master_group)}
Core:   #{instance_group_desc(@core_group)}
Task:   #{instance_group_desc(@task_group)}
HERE
end

#create_job_flow(job, emr_config = nil) ⇒ Object

Returns a configured job flow to the calling job.



12
13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/elasticrawl/cluster.rb', line 12

def create_job_flow(job, emr_config = nil)
  config = Config.new
  job_flow = Elasticity::JobFlow.new(config.access_key_id,
                                     config.secret_access_key)
  job_flow.name = "Job: #{job.job_name} #{job.job_desc}"
  job_flow.log_uri = job.log_uri

  configure_job_flow(job_flow)
  configure_instances(job_flow)
  configure_bootstrap_actions(job_flow, emr_config)

  job_flow
end