Class: Mortar::Local::Controller

Inherits:
Object
  • Object
show all
Includes:
InstallUtil
Defined in:
lib/mortar/local/controller.rb

Constant Summary collapse

NO_JAVA_ERROR_MESSAGE = 
<<EOF
A suitable java installation could not be found.  If you already have java installed
please set your JAVA_HOME environment variable before continuing.  Otherwise, a
suitable java installation will need to be added to your local system.

Installing Java
On OSX run `javac` from the command line.  This will intiate the installation.  For
Linux systems please consult the documentation on your relevant package manager.
EOF
NO_PYTHON_ERROR_MESSAGE = 
<<EOF
A suitable python installation could not be located.  Please ensure you have python 2.6+
installed on your local system.
EOF
NO_VIRTENV_ERROR_MESSAGE = 
<<EOF
A suitable Python installation was found, but it is required that virtualenv be installed
as well.  You can install it with pip, or download it directly from:
https://pypi.python.org/pypi/virtualenv
EOF
NO_AWS_KEYS_ERROR_MESSAGE = 
<<EOF
You have not set AWS access keys, which will often prevent you from accessing input data.  You can either:

- Login to your Mortar account to automatically sync your AWS keys from Mortar when running commands ("mortar login")

-  *or*, set your AWS keys via environment variables:

  export AWS_ACCESS_KEY="XXXXXXXXXXXX" 
  export AWS_SECRET_KEY="XXXXXXXXXXXX"

If your script does not need AWS S3 access, you can leave those values as XXXXXXXXXXXX.
EOF
API_CONFIG_ERROR_MESSAGE = 
<<EOF
We were unable to sync your AWS keys from Mortar.  
To continue, please specify your amazon AWS access key via environment variable AWS_ACCESS_KEY and your AWS secret key via environment variable AWS_SECRET_KEY, e.g.:
  
  export AWS_ACCESS_KEY="XXXXXXXXXXXX"
  export AWS_SECRET_KEY="XXXXXXXXXXXX"
  
If your script does not need AWS S3 access, you can set these variables to XXXXXXXXXXXX.
EOF

Instance Method Summary collapse

Methods included from InstallUtil

#download_file, #ensure_mortar_local_directory, #extract_tgz, #get_resource, #gitignore_template_path, #head_resource, #http_date_to_epoch, #install_date, #install_file_for, #is_newer_version, #jython_cache_directory, #jython_directory, #local_install_directory, #local_install_directory_name, #local_log_dir, #local_project_gitignore, #local_udf_log_dir, #make_call, #make_call_sleep_seconds, #note_install, #osx?, #project_root, #render_script_template, #reset_local_logs, #run_templated_script, #unset_hadoop_env_vars, #url_date

Methods included from Helpers

#action, #ask, #confirm, #copy_if_not_present_at_dest, #default_host, #deprecate, #display, #display_header, #display_object, #display_row, #display_table, #display_with_indent, #download_to_file, #ensure_dir_exists, #error, error_with_failure, error_with_failure=, extended, extended_into, #format_bytes, #format_date, #format_with_bang, #full_host, #get_terminal_environment, #home_directory, #host, #hprint, #hputs, included, included_into, #installed_with_omnibus?, #json_decode, #json_encode, #line_formatter, #longest, #output_with_bang, #pending_github_team_state_message, #quantify, #redisplay, #retry_on_exception, #running_on_a_mac?, #running_on_windows?, #set_buffer, #shell, #spinner, #status, #string_distance, #styled_array, #styled_error, #styled_hash, #styled_header, #suggestion, #test_name, #ticking, #time_ago, #truncate, #warning, #with_tty, #write_to_file

Instance Method Details

#ensure_local_install_dirs_in_gitignore ⇒ Object 



188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
 
# File 'lib/mortar/local/controller.rb', line 188

def ensure_local_install_dirs_in_gitignore()
  if File.exists? local_project_gitignore
    File.open(local_project_gitignore, 'r+') do |gitignore|
      contents = gitignore.read()
      gitignore.seek(0, IO::SEEK_END)

      unless contents[-1] == "\n"
        gitignore.puts "" # write a newline
      end

      unless contents.include? local_install_directory_name
        gitignore.puts local_install_directory_name
      end

      unless contents.include? "logs"
        gitignore.puts "logs"
      end

      unless contents.include? "illustrate-output"
        gitignore.puts "illustrate-output"
      end
    end
  end
end

#fetch_aws_keys(auth) ⇒ Object

Fetches AWS Keys based on auth



104
105
106
107
 
# File 'lib/mortar/local/controller.rb', line 104

def fetch_aws_keys(auth)
  project_name = ENV['MORTAR_PROJECT_NAME']
  return auth.api.get_config_vars(project_name).body['config']
end

#illustrate(pig_script, pig_alias, pig_version, pig_parameters, skip_pruning, no_browser) ⇒ Object

Main entry point for illustrating a pig alias



221
222
223
224
225
 
# File 'lib/mortar/local/controller.rb', line 221

def illustrate(pig_script, pig_alias, pig_version, pig_parameters, skip_pruning, no_browser)
  install_and_configure(pig_version, 'illustrate')
  pig = Mortar::Local::Pig.new()
  pig.illustrate_alias(pig_script, pig_alias, skip_pruning, no_browser, pig_version, pig_parameters)
end

#install_and_configure(pig_version = nil, command = nil, install_sqoop = false) ⇒ Object

Main entry point to perform installation and configuration necessary to run pig on the users local machine



121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/mortar/local/controller.rb', line 121

def install_and_configure(pig_version=nil, command=nil, install_sqoop=false)
  #To support old watchtower plugins we'll accept nil pig_version
  base = Mortar::Command::Base.new
  set_project_name(base)
  require_aws_keys()

  if pig_version.nil?
    pig_version = Mortar::PigVersion::Pig09.new
  end

  java = Mortar::Local::Java.new()
  unless java.check_install
    error(NO_JAVA_ERROR_MESSAGE)
  end

  pig = Mortar::Local::Pig.new()
  pig.install_or_update(pig_version, command)

  py = Mortar::Local::Python.new()
  unless py.check_or_install
    error(NO_PYTHON_ERROR_MESSAGE)
  end

  unless py.check_virtualenv
    error(NO_VIRTENV_ERROR_MESSAGE)
  end

  unless py.setup_project_python_environment
    msg = "\nUnable to setup a python environment with your dependencies, "
    msg += "see #{py.pip_error_log_path} for more details"
    error(msg)
  end

  jy = Mortar::Local::Jython.new()
  jy.install_or_update()

  if install_sqoop
    sqoop = Mortar::Local::Sqoop.new()
    sqoop.install_or_update()
  end

  write_local_readme

  ensure_local_install_dirs_in_gitignore
end

#repl(pig_version, pig_parameters) ⇒ Object 



233
234
235
236
237
 
# File 'lib/mortar/local/controller.rb', line 233

def repl(pig_version, pig_parameters)
  install_and_configure(pig_version, 'repl')
  pig = Mortar::Local::Pig.new()
  pig.launch_repl(pig_version, pig_parameters)
end

#require_aws_keys ⇒ Object

Asks to sync with AWS if user has not setup their aws keys



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/mortar/local/controller.rb', line 87

def require_aws_keys()        
  unless verify_aws_keys()
    auth = Mortar::Auth
    if !auth.has_credentials                      
      error(NO_AWS_KEYS_ERROR_MESSAGE)
    else
      vars = fetch_aws_keys(auth)
      if vars['aws_access_key_id'] && vars['aws_secret_access_key']
        set_aws_keys(vars['aws_access_key_id'], vars['aws_secret_access_key'])
      else
        error(API_CONFIG_ERROR_MESSAGE)
      end
    end 
  end
end

#run(pig_script, pig_version, pig_parameters) ⇒ Object

Main entry point for user running a pig script



214
215
216
217
218
 
# File 'lib/mortar/local/controller.rb', line 214

def run(pig_script, pig_version, pig_parameters)
  install_and_configure(pig_version, 'run')
  pig = Mortar::Local::Pig.new()
  pig.run_script(pig_script, pig_version, pig_parameters)
end

#run_luigi(pig_version, luigi_script, luigi_script_parameters, project_config_parameters) ⇒ Object 



239
240
241
242
243
244
245
246
 
# File 'lib/mortar/local/controller.rb', line 239

def run_luigi(pig_version, luigi_script, luigi_script_parameters, project_config_parameters)
  install_and_configure(pig_version, 'luigi')
  py = Mortar::Local::Python.new()
  unless py.run_stillson_luigi_client_cfg_expansion(luigi_script, project_config_parameters)
    error("Unable to expand your configuration template [luigiscripts/client.cfg.template] to [luigiscripts/client.cfg]")
  end
  py.run_luigi_script(luigi_script, luigi_script_parameters)
end

#set_aws_keys(aws_access_key, aws_secret_key) ⇒ Object 



109
110
111
112
 
# File 'lib/mortar/local/controller.rb', line 109

def set_aws_keys(aws_access_key, aws_secret_key)    
  ENV['AWS_ACCESS_KEY'] = aws_access_key
  ENV['AWS_SECRET_KEY'] = aws_secret_key    
end

#set_project_name(base) ⇒ Object 



114
115
116
117
118
 
# File 'lib/mortar/local/controller.rb', line 114

def set_project_name(base)
  project = base.project    
  project_name = base.options[:project] || project.name
  ENV['MORTAR_PROJECT_NAME'] = project_name
end

#sqoop_export_incremental(pig_version, connstr, dbtable, column, max_value, s3dest, options) ⇒ Object 



262
263
264
265
266
267
268
269
 
# File 'lib/mortar/local/controller.rb', line 262

def sqoop_export_incremental(pig_version, connstr, dbtable, column, max_value, s3dest, options)
  install_and_configure(pig_version, 'sqoop', true)
  sqoop = Mortar::Local::Sqoop.new()
  options[:dbtable] = dbtable
  options[:inc_column] = column
  options[:inc_value] = max_value
  sqoop.export(connstr, s3dest, options)
end

#sqoop_export_query(pig_version, connstr, query, s3dest, options) ⇒ Object 



255
256
257
258
259
260
 
# File 'lib/mortar/local/controller.rb', line 255

def sqoop_export_query(pig_version, connstr, query, s3dest, options)
  install_and_configure(pig_version, 'sqoop', true)
  sqoop = Mortar::Local::Sqoop.new()
  options[:sqlquery] = sqoop.prep_query(query)
  sqoop.export(connstr, s3dest, options)
end

#sqoop_export_table(pig_version, connstr, dbtable, s3dest, options) ⇒ Object 



248
249
250
251
252
253
 
# File 'lib/mortar/local/controller.rb', line 248

def sqoop_export_table(pig_version, connstr, dbtable, s3dest, options)
  install_and_configure(pig_version, 'sqoop', true)
  sqoop = Mortar::Local::Sqoop.new()
  options[:dbtable] = dbtable
  sqoop.export(connstr, s3dest, options)
end

#validate(pig_script, pig_version, pig_parameters) ⇒ Object 



227
228
229
230
231
 
# File 'lib/mortar/local/controller.rb', line 227

def validate(pig_script, pig_version, pig_parameters)
  install_and_configure(pig_version, 'validate')
  pig = Mortar::Local::Pig.new()
  pig.validate_script(pig_script, pig_version, pig_parameters)
end

#verify_aws_keys ⇒ Object

Checks if the user has properly specified their AWS keys



74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/mortar/local/controller.rb', line 74

def verify_aws_keys()
  if (not (ENV['AWS_ACCESS_KEY'] and ENV['AWS_SECRET_KEY'])) then
    if not ENV['MORTAR_IGNORE_AWS_KEYS']
      return false
    else
      return true
    end
  else
    return true
  end
end

#write_local_readme ⇒ Object 



167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
 
# File 'lib/mortar/local/controller.rb', line 167

def write_local_readme()
  readme_path = File.join(local_install_directory, "README")
  unless File.exists? readme_path
    file = File.new(readme_path, "w")
    file.write(<<-README
This directory is used by Mortar to install all of the necessary dependencies for
running mortar local commands.  You should not modify these files/directories as
they may be removed or updated at any time.

For additional Java dependencies you should place your jars in the root lib folder
of your project.  These jars will be automatically registered and 
available for use in your Pig scripts and UDFs.

You can specify additional Python dependencies in the requirements.txt file in 
the root of your project.
README
)
    file.close
  end
end