Class: Mortar::Local::Controller

Inherits:
Object
  • Object
show all
Includes:
InstallUtil
Defined in:
lib/mortar/local/controller.rb

Constant Summary collapse

NO_JAVA_ERROR_MESSAGE = 
<<EOF
A suitable java installation could not be found.  If you already have java installed
please set your JAVA_HOME environment variable before continuing.  Otherwise, a
suitable java installation will need to be added to your local system.

Installing Java
On OSX run `javac` from the command line.  This will intiate the installation.  For
Linux systems please consult the documentation on your relevant package manager.
EOF
NO_PYTHON_ERROR_MESSAGE = 
<<EOF
A suitable python installation could not be located.  Please ensure you have python 2.6+
installed on your local system.
EOF
NO_VIRTENV_ERROR_MESSAGE = 
<<EOF
A suitable Python installation was found, but it is required that virtualenv be installed
as well.  You can install it with pip, or download it directly from:
https://pypi.python.org/pypi/virtualenv
EOF
NO_AWS_KEYS_ERROR_MESSAGE = 
<<EOF
You have not set AWS access keys, which will often prevent you from accessing input data.  You can either:

- Login to your Mortar account to automatically sync your AWS keys from Mortar when running commands ("mortar login")

-  *or*, set your AWS keys via environment variables:

  export AWS_ACCESS_KEY="XXXXXXXXXXXX" 
  export AWS_SECRET_KEY="XXXXXXXXXXXX"

If your script does not need AWS S3 access, you can leave those values as XXXXXXXXXXXX.
EOF
API_CONFIG_ERROR_MESSAGE = 
<<EOF
We were unable to sync your AWS keys from Mortar.  
To continue, please specify your amazon AWS access key via environment variable AWS_ACCESS_KEY and your AWS secret key via environment variable AWS_SECRET_KEY, e.g.:
  
  export AWS_ACCESS_KEY="XXXXXXXXXXXX"
  export AWS_SECRET_KEY="XXXXXXXXXXXX"
  
If your script does not need AWS S3 access, you can set these variables to XXXXXXXXXXXX.
EOF

Instance Method Summary collapse

Methods included from InstallUtil

#download_file, #ensure_mortar_local_directory, #extract_tgz, #get_resource, #gitignore_template_path, #head_resource, #http_date_to_epoch, #install_date, #install_file_for, #is_newer_version, #jython_cache_directory, #jython_directory, #local_install_directory, #local_install_directory_name, #local_log_dir, #local_project_gitignore, #local_udf_log_dir, #make_call, #make_call_sleep_seconds, #note_install, #osx?, #project_root, #render_script_template, #reset_local_logs, #run_templated_script, #unset_hadoop_env_vars, #url_date

Methods included from Helpers

#action, #ask, #confirm, #copy_if_not_present_at_dest, #default_host, #deprecate, #display, #display_header, #display_object, #display_row, #display_table, #display_with_indent, #download_to_file, #ensure_dir_exists, #error, error_with_failure, error_with_failure=, extended, extended_into, #format_bytes, #format_date, #format_with_bang, #full_host, #get_terminal_environment, #home_directory, #host, #hprint, #hputs, included, included_into, #installed_with_omnibus?, #json_decode, #json_encode, #line_formatter, #longest, #output_with_bang, #pending_github_team_state_message, #quantify, #redisplay, #retry_on_exception, #running_on_a_mac?, #running_on_windows?, #set_buffer, #shell, #spinner, #status, #string_distance, #styled_array, #styled_error, #styled_hash, #styled_header, #suggestion, #test_name, #ticking, #time_ago, #truncate, #warning, #with_tty, #write_to_file

Instance Method Details

#ensure_local_install_dirs_in_gitignore ⇒ Object 



182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/mortar/local/controller.rb', line 182

def ensure_local_install_dirs_in_gitignore()
  if File.exists? local_project_gitignore
    File.open(local_project_gitignore, 'r+') do |gitignore|
      contents = gitignore.read()
      gitignore.seek(0, IO::SEEK_END)

      unless contents[-1] == "\n"
        gitignore.puts "" # write a newline
      end

      unless contents.include? local_install_directory_name
        gitignore.puts local_install_directory_name
      end

      unless contents.include? "logs"
        gitignore.puts "logs"
      end

      unless contents.include? "illustrate-output"
        gitignore.puts "illustrate-output"
      end
    end
  end
end

#fetch_aws_keys(auth) ⇒ Object

Fetches AWS Keys based on auth



103
104
105
106
 
# File 'lib/mortar/local/controller.rb', line 103

def fetch_aws_keys(auth)
  project_name = ENV['MORTAR_PROJECT_NAME']
  return auth.api.get_config_vars(project_name).body['config']
end

#illustrate(pig_script, pig_alias, pig_version, pig_parameters, skip_pruning, no_browser) ⇒ Object

Main entry point for illustrating a pig alias



215
216
217
218
219
 
# File 'lib/mortar/local/controller.rb', line 215

def illustrate(pig_script, pig_alias, pig_version, pig_parameters, skip_pruning, no_browser)
  install_and_configure(pig_version, 'illustrate')
  pig = Mortar::Local::Pig.new()
  pig.illustrate_alias(pig_script, pig_alias, skip_pruning, no_browser, pig_version, pig_parameters)
end

#install_and_configure(pig_version = nil, command = nil) ⇒ Object

Main entry point to perform installation and configuration necessary to run pig on the users local machine



120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
 
# File 'lib/mortar/local/controller.rb', line 120

def install_and_configure(pig_version=nil, command=nil)
  #To support old watchtower plugins we'll accept nil pig_version
  base = Mortar::Command::Base.new
  set_project_name(base)
  require_aws_keys()

  if pig_version.nil?
    pig_version = Mortar::PigVersion::Pig012Hadoop273.new
  end

  java = Mortar::Local::Java.new()
  unless java.check_install
    error(NO_JAVA_ERROR_MESSAGE)
  end

  pig = Mortar::Local::Pig.new()
  pig.install_or_update(pig_version, command)

  py = Mortar::Local::Python.new()
  unless py.check_or_install
    error(NO_PYTHON_ERROR_MESSAGE)
  end

  unless py.check_virtualenv
    error(NO_VIRTENV_ERROR_MESSAGE)
  end

  unless py.setup_project_python_environment
    msg = "\nUnable to setup a python environment with your dependencies, "
    msg += "see #{py.pip_error_log_path} for more details"
    error(msg)
  end

  jy = Mortar::Local::Jython.new()
  jy.install_or_update()

  write_local_readme

  ensure_local_install_dirs_in_gitignore
end

#repl(pig_version, pig_parameters) ⇒ Object 



227
228
229
230
231
 
# File 'lib/mortar/local/controller.rb', line 227

def repl(pig_version, pig_parameters)
  install_and_configure(pig_version, 'repl')
  pig = Mortar::Local::Pig.new()
  pig.launch_repl(pig_version, pig_parameters)
end

#require_aws_keys ⇒ Object

Asks to sync with AWS if user has not setup their aws keys



86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/mortar/local/controller.rb', line 86

def require_aws_keys()        
  unless verify_aws_keys()
    auth = Mortar::Auth
    if !auth.has_credentials                      
      error(NO_AWS_KEYS_ERROR_MESSAGE)
    else
      vars = fetch_aws_keys(auth)
      if vars['aws_access_key_id'] && vars['aws_secret_access_key']
        set_aws_keys(vars['aws_access_key_id'], vars['aws_secret_access_key'])
      else
        error(API_CONFIG_ERROR_MESSAGE)
      end
    end 
  end
end

#run(pig_script, pig_version, pig_parameters) ⇒ Object

Main entry point for user running a pig script



208
209
210
211
212
 
# File 'lib/mortar/local/controller.rb', line 208

def run(pig_script, pig_version, pig_parameters)
  install_and_configure(pig_version, 'run')
  pig = Mortar::Local::Pig.new()
  pig.run_script(pig_script, pig_version, pig_parameters)
end

#run_luigi(pig_version, luigi_script, luigi_script_parameters, project_config_parameters) ⇒ Object 



233
234
235
236
237
238
239
240
 
# File 'lib/mortar/local/controller.rb', line 233

def run_luigi(pig_version, luigi_script, luigi_script_parameters, project_config_parameters)
  install_and_configure(pig_version, 'luigi')
  py = Mortar::Local::Python.new()
  unless py.run_stillson_luigi_client_cfg_expansion(luigi_script, project_config_parameters)
    error("Unable to expand your configuration template [luigiscripts/client.cfg.template] to [luigiscripts/client.cfg]")
  end
  py.run_luigi_script(luigi_script, luigi_script_parameters)
end

#set_aws_keys(aws_access_key, aws_secret_key) ⇒ Object 



108
109
110
111
 
# File 'lib/mortar/local/controller.rb', line 108

def set_aws_keys(aws_access_key, aws_secret_key)    
  ENV['AWS_ACCESS_KEY'] = aws_access_key
  ENV['AWS_SECRET_KEY'] = aws_secret_key    
end

#set_project_name(base) ⇒ Object 



113
114
115
116
117
 
# File 'lib/mortar/local/controller.rb', line 113

def set_project_name(base)
  project = base.project    
  project_name = base.options[:project] || project.name
  ENV['MORTAR_PROJECT_NAME'] = project_name
end

#validate(pig_script, pig_version, pig_parameters) ⇒ Object 



221
222
223
224
225
 
# File 'lib/mortar/local/controller.rb', line 221

def validate(pig_script, pig_version, pig_parameters)
  install_and_configure(pig_version, 'validate')
  pig = Mortar::Local::Pig.new()
  pig.validate_script(pig_script, pig_version, pig_parameters)
end

#verify_aws_keys ⇒ Object

Checks if the user has properly specified their AWS keys



73
74
75
76
77
78
79
80
81
82
83
 
# File 'lib/mortar/local/controller.rb', line 73

def verify_aws_keys()
  if (not (ENV['AWS_ACCESS_KEY'] and ENV['AWS_SECRET_KEY'])) then
    if not ENV['MORTAR_IGNORE_AWS_KEYS']
      return false
    else
      return true
    end
  else
    return true
  end
end

#write_local_readme ⇒ Object 



161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/mortar/local/controller.rb', line 161

def write_local_readme()
  readme_path = File.join(local_install_directory, "README")
  unless File.exists? readme_path
    file = File.new(readme_path, "w")
    file.write(<<-README
This directory is used by Mortar to install all of the necessary dependencies for
running mortar local commands.  You should not modify these files/directories as
they may be removed or updated at any time.

For additional Java dependencies you should place your jars in the root lib folder
of your project.  These jars will be automatically registered and 
available for use in your Pig scripts and UDFs.

You can specify additional Python dependencies in the requirements.txt file in 
the root of your project.
README
)
    file.close
  end
end