Class: Traject::CommandLine

Inherits:
Object
  • Object
show all
Defined in:
lib/traject/command_line.rb

Overview

The class that executes for the Traject command line utility.

Warning, does do things like exit entire program on error at present. You probably don’t want to use this class for anything but an actual shell command line, if you want to execute indexing directly, just use the Traject::Indexer directly.

A CommandLine object has a single persistent Indexer object it uses

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(argv = ARGV) ⇒ CommandLine

Returns a new instance of CommandLine.



23
24
25
26
27
28
29
30
31
 
# File 'lib/traject/command_line.rb', line 23

def initialize(argv=ARGV)
  self.console = $stderr

  self.orig_argv      = argv.dup
  self.remaining_argv = argv

  self.slop    = create_slop!
  self.options = parse_options(self.remaining_argv)
end

Instance Attribute Details

#console ⇒ Object

Returns the value of attribute console.



21
22
23
 
# File 'lib/traject/command_line.rb', line 21

def console
  @console
end

#indexer ⇒ Object

Returns the value of attribute indexer.



20
21
22
 
# File 'lib/traject/command_line.rb', line 20

def indexer
  @indexer
end

#options ⇒ Object

Returns the value of attribute options.



19
20
21
 
# File 'lib/traject/command_line.rb', line 19

def options
  @options
end

#orig_argv ⇒ Object

orig_argv is origina one passed in, remaining_argv is after destructive processing by slop, still has file args in it etc.



18
19
20
 
# File 'lib/traject/command_line.rb', line 18

def orig_argv
  @orig_argv
end

#remaining_argv ⇒ Object

orig_argv is origina one passed in, remaining_argv is after destructive processing by slop, still has file args in it etc.



18
19
20
 
# File 'lib/traject/command_line.rb', line 18

def remaining_argv
  @remaining_argv
end

#slop ⇒ Object

Returns the value of attribute slop.



19
20
21
 
# File 'lib/traject/command_line.rb', line 19

def slop
  @slop
end

Instance Method Details

#arg_check! ⇒ Object 



208
209
210
211
212
213
214
215
216
 
# File 'lib/traject/command_line.rb', line 208

def arg_check!
  if options[:command] == "process" && (options[:conf].nil? || options[:conf].length == 0)
    self.console.puts "Error: Missing required configuration file"
    self.console.puts "Exiting..."
    self.console.puts
    self.console.puts self.slop.help
    exit 2
  end
end

#assemble_settings_hash(options) ⇒ Object 



237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
 
# File 'lib/traject/command_line.rb', line 237

def assemble_settings_hash(options)
  settings = {}

  # `-s key=value` command line
  (options[:setting] || []).each do |setting_pair|
    if setting_pair =~ /\A([^=]+)\=(.*)\Z/
      key, value = $1, $2
      settings[key] = value
    else
      self.console.puts "Unrecognized setting argument '#{setting_pair}':"
      self.console.puts "Should be of format -s key=value"
      exit 3
    end
  end

  # other command line shortcuts for settings
  if options[:debug]
    settings["log.level"] = "debug"
  end
  if options[:'debug-mode']
    require 'traject/debug_writer'
    settings["writer_class_name"] = "Traject::DebugWriter"
    settings["log.level"] = "debug"        
    settings["processing_thread_pool"] = 0
  end
  if options[:writer]
    settings["writer_class_name"] = options[:writer]
  end
  if options[:reader]
    settings["reader_class_name"] = options[:reader]
  end
  if options[:solr]
    settings["solr.url"] = options[:solr]
  end
  if options[:j]
    settings["writer_class_name"] = "JsonWriter"
    settings["json_writer.pretty_print"] = "true"
  end
  if options[:marc_type]
    settings["marc_source.type"] = options[:marc_type]
  end
  if options[:output_file]
    settings["output_file"] = options[:output_file]
  end

  return settings
end

#command_commit! ⇒ Object

Raises:

  • (ArgumentError) 


97
98
99
100
101
102
103
104
105
106
 
# File 'lib/traject/command_line.rb', line 97

def command_commit!
  require 'open-uri'
  raise ArgumentError.new("No solr.url setting provided") if indexer.settings['solr.url'].to_s.empty?

  url = "#{indexer.settings['solr.url']}/update?commit=true"
  indexer.logger.info("Sending commit to: #{url}")
  indexer.logger.info(  open(url).read )

  return true
end

#command_marcout!(io) ⇒ Object 



108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
 
# File 'lib/traject/command_line.rb', line 108

def command_marcout!(io)
  require 'marc'

  output_type = indexer.settings["marcout.type"].to_s
  output_type = "binary" if output_type.empty?

  output_arg      = unless indexer.settings["output_file"].to_s.empty?
    indexer.settings["output_file"]
  else
    $stdout
  end

  case output_type
  when "binary"
    writer = MARC::Writer.new(output_arg)

    allow_oversized = indexer.settings["marcout.allow_oversized"]
    if allow_oversized
      allow_oversized = (allow_oversized.to_s == "true") 
      writer.allow_oversized = allow_oversized
    end
  when "xml"
    writer = MARC::XMLWriter.new(output_arg)
  when "human"
    writer = output_arg.kind_of?(String) ? File.open(output_arg, "w:binary") : output_arg        
  else
    raise ArgumentError.new("traject marcout unrecognized marcout.type: #{output_type}")
  end

  reader      = indexer.reader!(io)

  reader.each do |record|
    writer.write record
  end

  writer.close

  return true
end

#create_slop! ⇒ Object 



286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
 
# File 'lib/traject/command_line.rb', line 286

def create_slop!
  return Slop.new(:strict => true) do
    banner "traject [options] -c configuration.rb [-c config2.rb] file.mrc"

    on 'v', 'version', "print version information to stderr"
    on 'd', 'debug', "Include debug log, -s log.level=debug"
    on 'h', 'help', "print usage information to stderr"
    on 'c', 'conf', 'configuration file path (repeatable)', :argument => true, :as => Array
    on :s, :setting, "settings: `-s key=value` (repeatable)", :argument => true, :as => Array
    on :r, :reader, "Set reader class, shortcut for -s reader_class_name=", :argument => true
    on :o, "output_file", "output file for Writer classes that write to files", :argument => true
    on :w, :writer, "Set writer class, shortcut for -s writer_class_name=", :argument => true
    on :u, :solr, "Set solr url, shortcut for -s solr.url=", :argument => true
    on :j, "output as pretty printed json, shortcut for -s writer_class_name=JsonWriter -s json_writer.pretty_print=true"
    on :t, :marc_type, "xml, json or binary. shortcut for -s marc_source.type=", :argument => true
    on :I, "load_path", "append paths to ruby $LOAD_PATH", :argument => true, :as => Array, :delimiter => ":"
    on :G, "Gemfile", "run with bundler and optionally specified Gemfile", :argument => :optional, :default => nil

    on :x, "command", "alternate traject command: process (default); marcout", :argument => true, :default => "process"

    on "stdin", "read input from stdin"
    on "debug-mode", "debug logging, single threaded, output human readable hashes"
  end
end

#execute ⇒ Object

Returns true on success or false on failure; may also raise exceptions; may also exit program directly itself (yeah, could use some normalization)



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/traject/command_line.rb', line 35

def execute
  # Do bundler setup FIRST to try and initialize all gems from gemfile
  # if requested.

  # have to use Slop object to tell diff between
  # no arg supplied and no option -g given at all
  if slop.present? :Gemfile
    require_bundler_setup(options[:Gemfile])
  end


  # We require them here instead of top of file,
  # so we have done bundler require before we require these.
  require 'traject'
  require 'traject/indexer'

  if options[:version]
    self.console.puts "traject version #{Traject::VERSION}"
    return
  end
  if options[:help]
    self.console.puts slop.help
    return
  end


  (options[:load_path] || []).each do |path|
    $LOAD_PATH << path unless $LOAD_PATH.include? path
  end

  arg_check!

  self.indexer = initialize_indexer!

  ######
  # SAFE TO LOG to indexer.logger starting here, after indexer is set up from conf files
  # with logging config.
  #####

  indexer.logger.info("traject executing with: `#{orig_argv.join(' ')}`")

  # Okay, actual command process! All command_ methods should return true
  # on success, or false on failure.
  result =
    case options[:command]
    when "process"
      (io, filename) = get_input_io(self.remaining_argv)
      indexer.settings['command_line.filename'] = filename if filename
      indexer.process(io)
    when "marcout"
       (io, filename) = get_input_io(self.remaining_argv)
      indexer.settings['command_line.filename'] = filename if filename
      command_marcout!(io)
    when "commit"
      command_commit!
    else
      raise ArgumentError.new("Unrecognized traject command: #{options[:command]}")
    end

  return result
end

#get_input_io(argv) ⇒ Object 



148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/traject/command_line.rb', line 148

def get_input_io(argv)
  # ARGF might be perfect for this, but problems with it include:
  # * jruby is broken, no way to set it's encoding, leads to encoding errors reading non-ascii
  #   https://github.com/jruby/jruby/issues/891
  # * It's apparently not enough like an IO object for at least one of the ruby-marc XML
  #   readers:
  #   NoMethodError: undefined method `to_inputstream' for ARGF:Object
  #      init at /Users/jrochkind/.gem/jruby/1.9.3/gems/marc-0.5.1/lib/marc/xml_parsers.rb:369
  #
  # * It INSISTS on reading from ARGFV, making it hard to test, or use when you want to give
  #   it a list of files on something other than ARGV.
  #
  # So for now we do just one file, or stdin if specified. Sorry!

  filename = nil
  if options[:stdin]
    indexer.logger.info("Reading from standard input")
    io = $stdin
  elsif argv.length > 1
    self.console.puts "Sorry, traject can only handle one input file at a time right now. `#{argv}` Exiting..."
    exit 1
  elsif argv.length == 0
    io = File.open(File::NULL, 'r')
    indexer.logger.info("Warning, no file input given. Use command-line argument '--stdin' to use standard input ")
  else
    io = File.open(argv.first, 'r')
    filename = argv.first
    indexer.logger.info "Reading from #{filename}"
  end
  
  return io, filename
end

#initialize_indexer! ⇒ Object 



311
312
313
314
315
316
 
# File 'lib/traject/command_line.rb', line 311

def initialize_indexer!
  indexer = Traject::Indexer.new self.assemble_settings_hash(self.options)
  load_configuration_files!(indexer, options[:conf])

  return indexer
end

#load_configuration_files!(my_indexer, conf_files) ⇒ Object 



181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
 
# File 'lib/traject/command_line.rb', line 181

def load_configuration_files!(my_indexer, conf_files)
  conf_files.each do |conf_path|
    begin
      file_io = File.open(conf_path)
    rescue Errno::ENOENT => e
      self.console.puts "Could not find configuration file '#{conf_path}', exiting..."
      exit 2
    end

    begin
      my_indexer.instance_eval(file_io.read, conf_path)
    rescue Exception => e
      if (conf_trace = e.backtrace.find {|l| l.start_with? conf_path}) &&
         (conf_trace =~ /\A.*\:(\d+)\:in/)
        line_number = $1
      end

      self.console.puts "Error processing configuration file '#{conf_path}' at line #{line_number}"
      self.console.puts "  #{e.class}: #{e.message}"
      if e.backtrace.first =~ /\A(.*)\:in/
        self.console.puts "  from #{$1}"
      end
      exit 3
    end
  end
end

#parse_options(argv) ⇒ Object 



318
319
320
321
322
323
324
325
326
327
328
329
330
331
 
# File 'lib/traject/command_line.rb', line 318

def parse_options(argv)

  begin
    self.slop.parse!(argv)
  rescue Slop::Error => e
    self.console.puts "Error: #{e.message}"
    self.console.puts "Exiting..."
    self.console.puts
    self.console.puts slop.help
    exit 1
  end

  return self.slop.to_hash
end

#require_bundler_setup(gemfile = nil) ⇒ Object

requires bundler/setup, optionally first setting ENV to tell bundler to use a specific gemfile. Gemfile arg can be relative to current working directory.



221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
 
# File 'lib/traject/command_line.rb', line 221

def require_bundler_setup(gemfile=nil)
  if gemfile
    # tell bundler what gemfile to use
    gem_path = File.expand_path( gemfile )
    # bundler not good at error reporting, we check ourselves
    unless File.exists? gem_path
      self.console.puts "Gemfile `#{gemfile}` does not exist, exiting..."
      self.console.puts
      self.console.puts slop.help
      exit 2
    end
    ENV["BUNDLE_GEMFILE"] = gem_path
  end
  require 'bundler/setup'
end