Class: Elasticrawl::Config

Inherits:
Object
  • Object
show all
Defined in:
lib/elasticrawl/config.rb

Overview

Represents the current configuration which is persisted to ~/.elasticrawl/ and contains 3 configuration files.

aws.yml - AWS access credentials unless stored in the environment

variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.

cluster.yml - Elastic MapReduce cluster config including instance groups. jobs.yml - Elastic MapReduce jobs config and the S3 bucket used for

storing data and logs.

This directory also contains the Elasticrawl SQLite database.

Constant Summary collapse

CONFIG_DIR = 
'.elasticrawl'
DATABASE_FILE = 
'elasticrawl.sqlite3'
TEMPLATES_DIR = 
'../../templates'
TEMPLATE_FILES = 
['aws.yml', 'cluster.yml', 'jobs.yml']

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(access_key_id = nil, secret_access_key = nil) ⇒ Config

Sets the AWS access credentials needed for the S3 and EMR API calls.



22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/elasticrawl/config.rb', line 22

def initialize(access_key_id = nil, secret_access_key = nil)
  # Credentials have been provided to the init command.
  @access_key_id = access_key_id
  @secret_access_key = secret_access_key

  # If credentials are not set then check if they are available in aws.yml.
  if dir_exists?
    config = load_config('aws')
    key = config['access_key_id']
    secret = config['secret_access_key']

    @access_key_id ||= key unless key == 'ACCESS_KEY_ID'
    @secret_access_key ||= secret unless secret == 'SECRET_ACCESS_KEY'
  end

  # If credentials are still not set then check AWS environment variables.
  @access_key_id ||= ENV['AWS_ACCESS_KEY_ID']
  @secret_access_key ||= ENV['AWS_SECRET_ACCESS_KEY']

  # Set AWS credentials for use when accessing the S3 API.
  AWS.config(:access_key_id => @access_key_id,
             :secret_access_key => @secret_access_key)
end

Instance Attribute Details

#access_key_id ⇒ Object (readonly)

Returns the value of attribute access_key_id.



18
19
20
 
# File 'lib/elasticrawl/config.rb', line 18

def access_key_id
  @access_key_id
end

#secret_access_key ⇒ Object (readonly)

Returns the value of attribute secret_access_key.



19
20
21
 
# File 'lib/elasticrawl/config.rb', line 19

def secret_access_key
  @secret_access_key
end

Instance Method Details

#access_key_prompt ⇒ Object

Displayed by init command.



139
140
141
142
143
144
 
# File 'lib/elasticrawl/config.rb', line 139

def access_key_prompt
  prompt = "Enter AWS Access Key ID:"
  prompt += " [#{@access_key_id}]" if @access_key_id.present?

  prompt
end

#bucket_exists?(bucket_name) ⇒ Boolean

Checks if a S3 bucket name is in use.

Returns:

  • (Boolean) 


96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/elasticrawl/config.rb', line 96

def bucket_exists?(bucket_name)
  begin
    s3 = AWS::S3.new
    s3.buckets[bucket_name].exists?

  rescue AWS::S3::Errors::SignatureDoesNotMatch => e
    raise AWSCredentialsInvalidError, 'AWS access credentials are invalid'
  rescue AWS::Errors::Base => s3e
    raise S3AccessError.new(s3e.http_response), s3e.message
  end
end

#config_dir ⇒ Object

Returns the location of the config directory.



47
48
49
 
# File 'lib/elasticrawl/config.rb', line 47

def config_dir
  File.join(Dir.home, CONFIG_DIR)
end

#create(bucket_name) ⇒ Object

Creates the S3 bucket and config directory. Deploys the config templates and creates the sqlite database.



110
111
112
113
114
115
116
 
# File 'lib/elasticrawl/config.rb', line 110

def create(bucket_name)
  create_bucket(bucket_name)
  deploy_templates(bucket_name)
  load_database

  status_message(bucket_name, 'created')
end

#delete ⇒ Object

Deletes the S3 bucket and config directory.



119
120
121
122
123
124
125
 
# File 'lib/elasticrawl/config.rb', line 119

def delete
  bucket_name = load_config('jobs')['s3_bucket_name']
  delete_bucket(bucket_name)
  delete_config_dir
  
  status_message(bucket_name, 'deleted')
end

#delete_warning ⇒ Object

Displayed by destroy command to confirm deletion.



128
129
130
131
132
133
134
135
136
 
# File 'lib/elasticrawl/config.rb', line 128

def delete_warning
  bucket_name = load_config('jobs')['s3_bucket_name']

  message = ['WARNING:']
  message << "Bucket s3://#{bucket_name} and its data will be deleted"
  message << "Config dir #{config_dir} will be deleted"

  message.join("\n")
end

#dir_exists? ⇒ Boolean

Checks if the configuration directory exists.

Returns:

  • (Boolean) 


52
53
54
 
# File 'lib/elasticrawl/config.rb', line 52

def dir_exists?
  Dir.exists?(config_dir)
end

#load_config(config_file) ⇒ Object

Loads a YAML configuration file.



57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/elasticrawl/config.rb', line 57

def load_config(config_file)
  if dir_exists?
    begin
      config_file = File.join(config_dir, "#{config_file}.yml")
      config = YAML::load(File.open(config_file))

    rescue StandardError => e
      raise FileAccessError, e.message
    end
  else
    raise ConfigDirMissingError, 'Config dir missing. Run init command'
  end
end

#load_database ⇒ Object

Loads the sqlite database. If no database exists it will be created and the database migrations will be run.



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/elasticrawl/config.rb', line 73

def load_database
  if dir_exists?
    config = {
      'adapter' => 'sqlite3',
      'database' => File.join(config_dir, DATABASE_FILE),
      'pool' => 5,
      'timeout' => 5000
    }

    begin
      ActiveRecord::Base.establish_connection(config)
      ActiveRecord::Migrator.migrate(File.join(File.dirname(__FILE__), \
        '../../db/migrate'), ENV['VERSION'] ? ENV['VERSION'].to_i : nil )

    rescue StandardError => e
      raise DatabaseAccessError, e.message
    end
  else
    raise ConfigDirMissingError, 'Config dir missing. Run init command'
  end
end

#secret_key_prompt ⇒ Object

Displayed by init command.



147
148
149
150
151
152
 
# File 'lib/elasticrawl/config.rb', line 147

def secret_key_prompt
  prompt = "Enter AWS Secret Access Key:"
  prompt += " [#{@secret_access_key}]" if @secret_access_key.present?

  prompt
end