Class: Mortar::Local::Controller

Inherits:
Object
  • Object
show all
Includes:
InstallUtil
Defined in:
lib/mortar/local/controller.rb

Constant Summary collapse

NO_JAVA_ERROR_MESSAGE = 
<<EOF
A suitable java installation could not be found.  If you already have java installed
please set your JAVA_HOME environment variable before continuing.  Otherwise, a
suitable java installation will need to be added to your local system.

Installing Java
On OSX run `javac` from the command line.  This will intiate the installation.  For
Linux systems please consult the documentation on your relevant package manager.
EOF
NO_PYTHON_ERROR_MESSAGE = 
<<EOF
A suitable python installation could not be located.  Please ensure you have python 2.6+
installed on your local system.
EOF
NO_VIRTENV_ERROR_MESSAGE = 
<<EOF
A suitable Python installation was found, but it is required that virtualenv be installed
as well.  You can install it with pip, or download it directly from:
https://pypi.python.org/pypi/virtualenv
EOF
NO_AWS_KEYS_ERROR_MESSAGE = 
<<EOF
You have not set AWS access keys, which will often prevent you from accessing input data.  You can either:

- Login to your Mortar account to automatically sync your AWS keys from Mortar when running commands ("mortar login")

-  *or*, set your AWS keys via environment variables:

  export AWS_ACCESS_KEY="XXXXXXXXXXXX" 
  export AWS_SECRET_KEY="XXXXXXXXXXXX"

If your script does not need AWS S3 access, you can leave those values as XXXXXXXXXXXX.
EOF
API_CONFIG_ERROR_MESSAGE = 
<<EOF
We were unable to sync your AWS keys from Mortar.  
To continue, please specify your amazon AWS access key via environment variable AWS_ACCESS_KEY and your AWS secret key via environment variable AWS_SECRET_KEY, e.g.:
  
  export AWS_ACCESS_KEY="XXXXXXXXXXXX"
  export AWS_SECRET_KEY="XXXXXXXXXXXX"
  
If your script does not need AWS S3 access, you can set these variables to XXXXXXXXXXXX.
EOF

Instance Method Summary collapse

Methods included from InstallUtil

#download_file, #ensure_mortar_local_directory, #extract_tgz, #get_resource, #gitignore_template_path, #head_resource, #http_date_to_epoch, #install_date, #install_file_for, #is_newer_version, #jython_cache_directory, #jython_directory, #local_install_directory, #local_install_directory_name, #local_log_dir, #local_project_gitignore, #local_udf_log_dir, #make_call, #make_call_sleep_seconds, #note_install, #osx?, #project_root, #render_script_template, #reset_local_logs, #run_templated_script, #unset_hadoop_env_vars, #url_date

Methods included from Helpers

#action, #ask, #confirm, #copy_if_not_present_at_dest, #default_host, #deprecate, #display, #display_header, #display_object, #display_row, #display_table, #display_with_indent, #download_to_file, #ensure_dir_exists, #error, error_with_failure, error_with_failure=, extended, extended_into, #format_bytes, #format_date, #format_with_bang, #get_terminal_environment, #home_directory, #host, #hprint, #hputs, included, included_into, #installed_with_omnibus?, #json_decode, #json_encode, #line_formatter, #longest, #output_with_bang, #quantify, #redisplay, #retry_on_exception, #running_on_a_mac?, #running_on_windows?, #set_buffer, #shell, #spinner, #status, #string_distance, #styled_array, #styled_error, #styled_hash, #styled_header, #suggestion, #test_name, #ticking, #time_ago, #truncate, #warning, #with_tty, #write_to_file

Instance Method Details

#ensure_local_install_dirs_in_gitignore ⇒ Object 



150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/mortar/local/controller.rb', line 150

def ensure_local_install_dirs_in_gitignore()
  if File.exists? local_project_gitignore
    File.open(local_project_gitignore, 'r+') do |gitignore|
      contents = gitignore.read()
      gitignore.seek(0, IO::SEEK_END)

      unless contents[-1] == "\n"
        gitignore.puts "" # write a newline
      end

      unless contents.include? local_install_directory_name
        gitignore.puts local_install_directory_name
      end

      unless contents.include? "logs"
        gitignore.puts "logs"
      end

      unless contents.include? "illustrate-output"
        gitignore.puts "illustrate-output"
      end
    end
  end
end

#fetch_aws_keys(auth, base) ⇒ Object

Fetches AWS Keys based on auth



103
104
105
106
107
 
# File 'lib/mortar/local/controller.rb', line 103

def fetch_aws_keys(auth, base)    
  project = base.project    
  project_name = base.options[:project] || project.name    
  return auth.api.get_config_vars(project_name).body['config']
end

#illustrate(pig_script, pig_alias, pig_version, pig_parameters, skip_pruning, no_browser) ⇒ Object

Main entry point for illustrating a pig alias



184
185
186
187
188
189
 
# File 'lib/mortar/local/controller.rb', line 184

def illustrate(pig_script, pig_alias, pig_version, pig_parameters, skip_pruning, no_browser)
  require_aws_keys
  install_and_configure(pig_version, 'illustrate')
  pig = Mortar::Local::Pig.new()
  pig.illustrate_alias(pig_script, pig_alias, skip_pruning, no_browser, pig_version, pig_parameters)
end

#install_and_configure(pig_version = nil, command = nil) ⇒ Object

Main entry point to perform installation and configuration necessary to run pig on the users local machine



116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/mortar/local/controller.rb', line 116

def install_and_configure(pig_version=nil, command=nil)
  #To support old watchtower plugins we'll accept nil pig_version
  if pig_version.nil?
    pig_version = Mortar::PigVersion::Pig09.new
  end
  java = Mortar::Local::Java.new()
  unless java.check_install
    error(NO_JAVA_ERROR_MESSAGE)
  end

  pig = Mortar::Local::Pig.new()
  pig.install_or_update(pig_version, command)

  py = Mortar::Local::Python.new()
  unless py.check_or_install
    error(NO_PYTHON_ERROR_MESSAGE)
  end

  unless py.check_virtualenv
    error(NO_VIRTENV_ERROR_MESSAGE)
  end

  unless py.setup_project_python_environment
    msg = "\nUnable to setup a python environment with your dependencies, "
    msg += "see #{py.pip_error_log_path} for more details"
    error(msg)
  end

  jy = Mortar::Local::Jython.new()
  jy.install_or_update()

  ensure_local_install_dirs_in_gitignore
end

#repl(pig_version, pig_parameters) ⇒ Object 



197
198
199
200
201
 
# File 'lib/mortar/local/controller.rb', line 197

def repl(pig_version, pig_parameters)
  install_and_configure(pig_version, 'repl')
  pig = Mortar::Local::Pig.new()
  pig.launch_repl(pig_version, pig_parameters)
end

#require_aws_keys ⇒ Object

Asks to sync with AWS if user has not setup their aws keys



86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/mortar/local/controller.rb', line 86

def require_aws_keys()        
  unless verify_aws_keys()
    auth = Mortar::Auth
    if !auth.has_credentials                      
      error(NO_AWS_KEYS_ERROR_MESSAGE)
    else
      vars = fetch_aws_keys(auth, Mortar::Command::Base.new)
      if vars['aws_access_key_id'] && vars['aws_secret_access_key']
        set_aws_keys(vars['aws_access_key_id'], vars['aws_secret_access_key'])
      else
        error(API_CONFIG_ERROR_MESSAGE)
      end
    end 
  end
end

#run(pig_script, pig_version, pig_parameters) ⇒ Object

Main entry point for user running a pig script



176
177
178
179
180
181
 
# File 'lib/mortar/local/controller.rb', line 176

def run(pig_script, pig_version, pig_parameters)
  require_aws_keys
  install_and_configure(pig_version, 'run')
  pig = Mortar::Local::Pig.new()
  pig.run_script(pig_script, pig_version, pig_parameters)
end

#run_luigi(luigi_script, user_script_args) ⇒ Object 



203
204
205
206
207
 
# File 'lib/mortar/local/controller.rb', line 203

def run_luigi(luigi_script, user_script_args)
  install_and_configure(nil, 'luigi')
  py = Mortar::Local::Python.new()
  py.run_luigi_script(luigi_script, user_script_args)
end

#set_aws_keys(aws_access_key, aws_secret_key) ⇒ Object 



109
110
111
112
 
# File 'lib/mortar/local/controller.rb', line 109

def set_aws_keys(aws_access_key, aws_secret_key)    
  ENV['AWS_ACCESS_KEY'] = aws_access_key
  ENV['AWS_SECRET_KEY'] = aws_secret_key    
end

#validate(pig_script, pig_version, pig_parameters) ⇒ Object 



191
192
193
194
195
 
# File 'lib/mortar/local/controller.rb', line 191

def validate(pig_script, pig_version, pig_parameters)
  install_and_configure(pig_version, 'validate')
  pig = Mortar::Local::Pig.new()
  pig.validate_script(pig_script, pig_version, pig_parameters)
end

#verify_aws_keys ⇒ Object

Checks if the user has properly specified their AWS keys



73
74
75
76
77
78
79
80
81
82
83
 
# File 'lib/mortar/local/controller.rb', line 73

def verify_aws_keys()
  if (not (ENV['AWS_ACCESS_KEY'] and ENV['AWS_SECRET_KEY'])) then
    if not ENV['MORTAR_IGNORE_AWS_KEYS']
      return false
    else
      return true
    end
  else
    return true
  end
end