Class: Mortar::Command::Local

Inherits:
Base
  • Object
show all
Defined in:
lib/mortar/command/local.rb

Overview

run select pig commands on your local machine

Instance Attribute Summary

Attributes inherited from Base

#args, #options

Instance Method Summary collapse

Methods inherited from Base

#api, #ask_public, #config_parameters, #get_error_message_context, #git, #initialize, #initialize_embedded_project, #luigi_parameters, namespace, #pig_parameters, #project, #register_api_call, #register_do, #register_project, #spark_script_arguments, #validate_project_name, #validate_project_structure

Methods included from Helpers

#action, #ask, #confirm, #copy_if_not_present_at_dest, #default_host, #deprecate, #display, #display_header, #display_object, #display_row, #display_table, #display_with_indent, #download_to_file, #ensure_dir_exists, #error, error_with_failure, error_with_failure=, extended, extended_into, #format_bytes, #format_date, #format_with_bang, #full_host, #get_terminal_environment, #home_directory, #host, #hprint, #hputs, included, included_into, #installed_with_omnibus?, #json_decode, #json_encode, #line_formatter, #longest, #output_with_bang, #pending_github_team_state_message, #quantify, #redisplay, #retry_on_exception, #running_on_a_mac?, #running_on_windows?, #set_buffer, #shell, #spinner, #status, #string_distance, #styled_array, #styled_error, #styled_hash, #styled_header, #suggestion, #test_name, #ticking, #time_ago, #truncate, #warning, #with_tty, #write_to_file

Constructor Details

This class inherits a constructor from Mortar::Command::Base

Instance Method Details

#characterize ⇒ Object

local:characterize -f PARAMFILE

Characterize will inspect your input data, inferring a schema and

generating keys, if needed. It will output CSV containing various
statistics about your data (most common values, percent null, etc.)

-f, –param-file PARAMFILE # Load pig parameter values from a file -g, –pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.

Load some data and emit statistics. PARAMFILE (Required):

LOADER=<full class path of loader function>
INPUT_SRC=<Location of the input data>
OUTPUT_PATH=<Relative path from project root for output>
INFER_TYPES=<when true, recursively infers types for input data>

Example paramfile:

LOADER=org.apache.pig.piggybank.storage.JsonLoader()
INPUT_SRC=s3n://twitter-gardenhose-mortar/example
OUTPUT_PATH=twitter_char
INFER_TYPES=true



102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/mortar/command/local.rb', line 102

def characterize
  validate_arguments!

  unless options[:param_file]
    error("Usage: mortar local:characterize -f PARAMFILE.\nMust specify parameter file. For detailed help run:\n\n   mortar local:characterize -h")
  end

  #cd into the project root
  project_root = options[:project_root] ||= Dir.getwd
  unless File.directory?(project_root)
    error("No such directory #{project_root}")
  end

  Dir.chdir(project_root)

  gen = Mortar::Generators::CharacterizeGenerator.new
  gen.generate_characterize

  controlscript_name = "controlscripts/lib/characterize_control.py"
  gen = Mortar::Generators::CharacterizeGenerator.new
  gen.generate_characterize
  script = validate_script!(controlscript_name)
  params = config_parameters.concat(pig_parameters)

  ctrl = Mortar::Local::Controller.new
  ctrl.run(script, pig_version, params)
  gen.cleanup_characterize(project_root)
end

#configure ⇒ Object

local:configure

Install dependencies for running this mortar project locally - other mortar:local commands will also perform this step automatically.

-g, –pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>. –project-root PROJECTDIR # The root directory of the project if not the CWD



33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/mortar/command/local.rb', line 33

def configure
  validate_arguments!

  # cd into the project root
  project_root = options[:project_root] ||= Dir.getwd
  unless File.directory?(project_root)
    error("No such directory #{project_root}")
  end
  Dir.chdir(project_root)

  ctrl = Mortar::Local::Controller.new
  ctrl.install_and_configure(pig_version, nil, true)
end

#illustrate ⇒ Object

local:illustrate PIGSCRIPT [ALIAS]

Locally illustrate the effects and output of a pigscript. If an alias is specified, will show data flow from the ancestor LOAD statements to the alias itself. If no alias is specified, will show data flow through all aliases in the script.

-s, –skippruning # Don’t try to reduce the illustrate results to the smallest size possible. -p, –parameter NAME=VALUE # Set a pig parameter value in your script. -f, –param-file PARAMFILE # Load pig parameter values from a file. -g, –pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>. –no_browser # Don’t open the illustrate results automatically in the browser. –project-root PROJECTDIR # The root directory of the project if not the CWD

Examples:

Illustrate all relations in the generate_regression_model_coefficients pigscript:
    $ mortar illustrate pigscripts/generate_regression_model_coefficients.pig



148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
 
# File 'lib/mortar/command/local.rb', line 148

def illustrate
  pigscript_name = shift_argument
  alias_name = shift_argument
  validate_arguments!

  skip_pruning = options[:skippruning] ||= false
  no_browser = options[:no_browser] ||= false

  unless pigscript_name
    error("Usage: mortar local:illustrate PIGSCRIPT [ALIAS]\nMust specify PIGSCRIPT.")
  end

  # cd into the project root
  project_root = options[:project_root] ||= Dir.getwd
  unless File.directory?(project_root)
    error("No such directory #{project_root}")
  end
  Dir.chdir(project_root)

  pigscript = validate_pigscript!(pigscript_name)
  params = config_parameters.concat(pig_parameters)

  ctrl = Mortar::Local::Controller.new
  ctrl.illustrate(pigscript, alias_name, pig_version, params, skip_pruning, no_browser)
end

#luigi ⇒ Object

local:luigi SCRIPT

Run a luigi pipeline script on your local machine in local scheduler mode. Any additional command line arguments will be passed directly to the luigi script.

–project-root PROJECTDIR # The root directory of the project if not the CWD -p, –parameter NAME=VALUE # [deprecated] Instead, pass luigi parameters directly as options (see below) -f, –param-file PARAMFILE # [deprecated] Instead, pass luigi parameters directly as options (see below)

Examples:

Run the recsys luigi script with a parameter named date-interval
    $ mortar local:luigi luigiscripts/recsys.py --date-interval 2012-04



240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
 
# File 'lib/mortar/command/local.rb', line 240

def luigi
  script_name = shift_argument
  unless script_name
    error("Usage: mortar local:luigi SCRIPT\nMust specify SCRIPT.")
  end

  # cd into the project root
  project_root = options[:project_root] ||= Dir.getwd
  unless File.directory?(project_root)
    error("No such directory #{project_root}")
  end
  Dir.chdir(project_root)

  script = validate_luigiscript!(script_name)

  #Set git ref as environment variable for mortar-luigi to use when
  #running a MortarTask
  git_ref = sync_code_with_cloud()
  ENV['MORTAR_LUIGI_GIT_REF'] = git_ref

  # pick up standard luigi-style params provided by the user
  luigi_cli_parameters = luigi_parameters()

  # pick up old pig-style parameters (included for backwards compatibility)
  pig_style_parameters = pig_parameters()
  if pig_style_parameters.length > 0
    warn "[DEPRECATION] Passing luigi parameters with -p is deprecated. Please pass them directly (e.g. --myparam myvalue)"
  end

  luigi_cli_parameters.concat(pig_style_parameters)
  cli_parameters = \
    luigi_cli_parameters.sort_by { |p| p['name'] }.map { |arg| ["--#{arg['name']}", "#{arg['value']}"] }.flatten

  # get project configuration parameters
  project_config_params = config_parameters()

  ctrl = Mortar::Local::Controller.new
  ctrl.run_luigi(pig_version, script, cli_parameters, project_config_params)
end

#repl ⇒ Object

local:repl

Start a local Pig REPL session -p, –parameter NAME=VALUE # Set a pig parameter value in your script. -f, –param-file PARAMFILE # Load pig parameter values from a file. -g, –pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>.



217
218
219
220
221
222
223
224
 
# File 'lib/mortar/command/local.rb', line 217

def repl
  validate_arguments!

  params = config_parameters.concat(pig_parameters)

  ctrl = Mortar::Local::Controller.new
  ctrl.repl(pig_version, params)
end

#run ⇒ Object

local:run SCRIPT

Run a job on your local machine.

-p, –parameter NAME=VALUE # Set a pig parameter value in your script. -f, –param-file PARAMFILE # Load pig parameter values from a file. -g, –pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>. –project-root PROJECTDIR # The root directory of the project if not the CWD

Examples:

Run the generate_regression_model_coefficients script locally.
    $ mortar local:run pigscripts/generate_regression_model_coefficients.pig



60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/mortar/command/local.rb', line 60

def run
  script_name = shift_argument
  unless script_name
    error("Usage: mortar local:run SCRIPT\nMust specify SCRIPT.")
  end
  validate_arguments!

  # cd into the project root
  project_root = options[:project_root] ||= Dir.getwd
  unless File.directory?(project_root)
    error("No such directory #{project_root}")
  end
  Dir.chdir(project_root)
  script = validate_script!(script_name)
  params = config_parameters.concat(pig_parameters)

  ctrl = Mortar::Local::Controller.new
  ctrl.run(script, pig_version, params)
end

#sqoop_incremental ⇒ Object

local:sqoop_incremental dbtype database-name table column value s3-destination

Export all records where column is > value

-h, –host HOSTNAME # Database host, localhost assumed if not specified -u, –username USERNAME # User to log into the database with -p, –password PASSWORD # Password to log into the database -j, –jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class -d, –direct # Use a direct import path -r, –driverjar JARFILE # Path to the jar containing the jdbc driver -c, –jdbcconnectionstring CONNECTION_STRING # The JDBC connection string override

Examples:

Export from the newest users
    $ mortar local:sqoop_incremental postgres mydb users user_id 12345 s3://com.dbhost01/archive



380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
 
# File 'lib/mortar/command/local.rb', line 380

def sqoop_incremental
  dbtype = shift_argument
  unless dbtype
    error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify database type.")
  end
  physdb = shift_argument
  unless physdb
    error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify database name.")
  end
  table = shift_argument
  unless table
    error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify database table.")
  end
  column = shift_argument
  unless column
    error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify column.")
  end
  max_value = shift_argument
  unless max_value
    error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify value.")
  end
  s3dest = shift_argument
  unless s3dest
    error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify s3 destination.")
  end
  validate_arguments!

  dbhost = options[:host] || "localhost"
  connstr = Shellwords.escape(options[:jdbcconnectionstring] || jdbc_conn(dbtype, dbhost, physdb))

  ctrl = Mortar::Local::Controller.new
  ctrl.sqoop_export_incremental(pig_version, connstr, table, column, max_value, s3dest, options)
end

#sqoop_query ⇒ Object

local:sqoop_query dbtype database-name query s3-destination

Export the result of an SQL query to S3.

-h, –host HOSTNAME # Database host, localhost assumed if not specified -u, –username USERNAME # User to log into the database with -p, –password PASSWORD # Password to log into the database -j, –jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class -d, –direct # Use a direct import path -r, –driverjar JARFILE # Path to the jar containing the jdbc driver -c, –jdbcconnectionstring CONNECTION_STRING # The JDBC connection string override

Examples:

Export from a postgres database
    $ mortar local:sqoop_query postgres mydb "select user_name, id from users" s3://com.dbhost01/archive



338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
 
# File 'lib/mortar/command/local.rb', line 338

def sqoop_query
  dbtype = shift_argument
  unless dbtype
    error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify database type.")
  end
  physdb = shift_argument
  unless physdb
    error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify database name.")
  end
  query = shift_argument
  unless query
    error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify sql query.")
  end
  s3dest = shift_argument
  unless s3dest
    error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify s3 destination.")
  end
  validate_arguments!

  dbhost = options[:host] || "localhost"
  connstr = Shellwords.escape(options[:jdbcconnectionstring] || jdbc_conn(dbtype, dbhost, physdb))

  ctrl = Mortar::Local::Controller.new
  ctrl.sqoop_export_query(pig_version, connstr, query, s3dest, options)
end

#sqoop_table ⇒ Object

local:sqoop_table dbtype database-name table s3-destination

Export data from an RDBMS table to S3.

-h, –host HOSTNAME # Database host, localhost assumed if not specified -u, –username USERNAME # User to log into the database with -p, –password PASSWORD # Password to log into the database -j, –jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class -d, –direct # Use a direct import path -r, –driverjar JARFILE # Path to the jar containing the jdbc driver -c, –jdbcconnectionstring CONNECTION_STRING # The JDBC connection string override

Examples:

Export from a postgres database
    $ mortar local:sqoop_table postgres mydb mytable s3://com.dbhost01/archive -u steve -p stevespassword



296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
 
# File 'lib/mortar/command/local.rb', line 296

def sqoop_table
  dbtype = shift_argument
  unless dbtype
    error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify database type.")
  end
  physdb = shift_argument
  unless physdb
    error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify database name.")
  end
  dbtable = shift_argument
  unless dbtable
    error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify database table.")
  end
  s3dest = shift_argument
  unless s3dest
    error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify s3 destination.")
  end
  validate_arguments!

  dbhost = options[:host] || "localhost"
  connstr = Shellwords.escape(options[:jdbcconnectionstring] || jdbc_conn(dbtype, dbhost, physdb))

  ctrl = Mortar::Local::Controller.new
  ctrl.sqoop_export_table(pig_version, connstr, dbtable, s3dest, options)
end

#validate ⇒ Object

local:validate SCRIPT

Locally validate the syntax of a script.

-p, –parameter NAME=VALUE # Set a pig parameter value in your script. -f, –param-file PARAMFILE # Load pig parameter values from a file. -g, –pigversion PIG_VERSION # Set pig version. Options are <PIG_VERSION_OPTIONS>. –project-root PROJECTDIR # The root directory of the project if not the CWD

Examples:

Check the pig syntax of the generate_regression_model_coefficients pigscript locally.
    $ mortar local:validate pigscripts/generate_regression_model_coefficients.pig



188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
 
# File 'lib/mortar/command/local.rb', line 188

def validate
  script_name = shift_argument
  unless script_name
    error("Usage: mortar local:validate SCRIPT\nMust specify SCRIPT.")
  end
  validate_arguments!

  # cd into the project root
  project_root = options[:project_root] ||= Dir.getwd
  unless File.directory?(project_root)
    error("No such directory #{project_root}")
  end
  Dir.chdir(project_root)

  script = validate_script!(script_name)
  params = config_parameters.concat(pig_parameters)

  ctrl = Mortar::Local::Controller.new
  ctrl.validate(script, pig_version, params)
end