Module: Embulk

Defined in:
lib/embulk.rb,
lib/embulk/page.rb,
lib/embulk/error.rb,
lib/embulk/buffer.rb,
lib/embulk/column.rb,
lib/embulk/plugin.rb,
lib/embulk/schema.rb,
lib/embulk/version.rb,
lib/embulk/guess_csv.rb,
lib/embulk/guess_gzip.rb,
lib/embulk/data_source.rb,
lib/embulk/guess_plugin.rb,
lib/embulk/input_plugin.rb,
lib/embulk/page_builder.rb,
lib/embulk/guess_charset.rb,
lib/embulk/guess_newline.rb,
lib/embulk/output_plugin.rb,
lib/embulk/java/bootstrap.rb,
lib/embulk/plugin_registry.rb,
lib/embulk/java/time_helper.rb,
lib/embulk/command/embulk_run.rb,
lib/embulk/command/embulk_example.rb,
lib/embulk/command/embulk_generate_bin.rb,
lib/embulk/data/bundle/embulk/input_example.rb,
lib/embulk/data/bundle/embulk/output_example.rb

Defined Under Namespace

Modules: Java, TimeFormatGuess, Type Classes: Buffer, Column, ConfigError, DataSource, GuessCharset, GuessCsv, GuessNewline, GuessPlugin, GzipGuess, InputExample, InputPlugin, LineGuessPlugin, OutputExample, OutputPlugin, Page, PageBuilder, PluginManager, PluginRegistry, Schema, TextGuessPlugin

Constant Summary collapse

Plugin = 
PluginManager.new
VERSION = 
"0.2.0"

Class Method Summary collapse

Class Method Details

.b(s) ⇒ Object 



59
60
61
 
# File 'lib/embulk/command/embulk_generate_bin.rb', line 59

def self.b(s)
  s.force_encoding('ASCII-8BIT')
end

.create_example(path) ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/embulk/command/embulk_example.rb', line 2

def self.create_example(path)
  require 'fileutils'
  require 'zlib'

  puts "  Creating #{path}/"
  FileUtils.mkdir_p File.join(path, 'csv')
  puts "  Creating #{path}/csv/"

  puts "  Creating #{path}/csv/sample_01.csv.gz"
  Zlib::GzipWriter.open(File.join(path, 'csv', 'sample_01.csv.gz')) do |f|
    f.write <<EOF
id,account,time,purchase,comment
1,32864,2015-01-27 19:23:49,20150127,embulk
2,14824,2015-01-27 19:01:23,20150127,embulk jruby
3,27559,2015-01-28 02:20:02,20150128,embulk core
4,11270,2015-01-29 11:54:36,20150129,"Embulk ""csv"" parser plugin"
EOF
  end

  puts "  Creating #{path}/example.yml"
  File.open(File.join(path, 'example.yml'), 'w') do |f|
    f.write <<EOF
in:
type: file
paths: ["#{File.expand_path File.join(path, 'csv')}"]
out:
type: stdout
EOF
  end
end

.generate_bin(options = {}) ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
 
# File 'lib/embulk/command/embulk_generate_bin.rb', line 2

def self.generate_bin(options={})
  jruby_jar_path = org.jruby.Main.java_class.protection_domain.code_source.location.to_s
  if __FILE__ =~ /^classpath:/ || __FILE__.include?('!/')
    resource_class = org.embulk.command.Runner.java_class
    ruby_script_path = resource_class.resource("/embulk/command/embulk.rb").to_s
  else
    ruby_script_path = File.join(File.dirname(__FILE__), 'embulk.rb')
  end
  java_home = java.lang.System.properties['java.home']

  generate_bin_data(jruby_jar_path, ruby_script_path, options.merge(java_home: java_home))
end

.generate_bin_data(jruby_jar_path, ruby_script_path, options = {}) ⇒ Object 



15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/embulk/command/embulk_generate_bin.rb', line 15

def self.generate_bin_data(jruby_jar_path, ruby_script_path, options={})
  if java_home = options[:java_home]
    java_home_script = %{export JAVA_HOME='#{java_home}'}
    java_path = %{"$JAVA_HOME"/bin/java}
  else
    java_home_script = %{}
    java_path = %{java}
  end

  # TODO parse -D options to set them to java
  shell_script = <<EOF
#!/bin/sh
=begin 2>/dev/null
#{java_home_script}
exec #{java_path} -classpath "$0" org.jruby.Main "$0" "$@"
exit 127
=end
EOF

  if options[:bundle_path] == :here
    bundle_path_script = %{ENV['EMBULK_BUNDLE_PATH'] = File.expand_path('..', File.dirname(__FILE__))}
  elsif path = options[:bundle_path]
    bundle_path_script = %{ENV['EMBULK_BUNDLE_PATH'] = '#{path}'}
  else
    bundle_path_script = b ''
  end

  ruby_init_script = b <<EOF
#{bundle_path_script}
ENV.delete 'GEM_HOME'
ENV.delete 'GEM_PATH'
EOF

  ruby_script = b(File.read(ruby_script_path))
  if i = ruby_script.index(b("\n__END__\n"))
    # delete contents after __END__
    ruby_script = ruby_script[0, i]
  end

  jruby_jar = b(File.read(jruby_jar_path))

  data = shell_script << ruby_init_script + ruby_script << b("\n__END__\n") << jruby_jar
end

.home(dir) ⇒ Object 



199
200
201
202
 
# File 'lib/embulk/command/embulk_run.rb', line 199

def self.home(dir)
  home = File.expand_path('../../..', File.dirname(__FILE__))
  File.join(home, dir)
end

.java? ⇒ Boolean

Returns:

  • (Boolean) 


3
4
5
 
# File 'lib/embulk.rb', line 3

def self.java?
  true
end

.run(argv) ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
 
# File 'lib/embulk/command/embulk_run.rb', line 2

def self.run(argv)
  # default_bundle_path
  default_bundle_path = nil
  gemfile_path = ENV['BUNDLE_GEMFILE'].to_s
  gemfile_path = nil if gemfile_path.empty?
  default_bundle_path = File.dirname(gemfile_path) if gemfile_path

  # default GEM_HOME is ~/.embulk/jruby/1.9/. If -b option is set,
  # GEM_HOME is already set by embulk/command/embulk.rb
  gem_home = ENV['GEM_HOME'].to_s
  if gem_home.empty?
    ENV['GEM_HOME'] = File.expand_path File.join(ENV['HOME'], '.embulk', Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
    Gem.clear_paths  # force rubygems to reload GEM_HOME
  end

  i = argv.find_index {|arg| arg !~ /^\-/ }
  usage nil unless i
  subcmd = argv.slice!(i)

  load_paths = []
  options = {}

  # to make sure org.embulk.jruby.JRubyScriptingModule can require 'embulk/java/bootstrap'
  $LOAD_PATH << Embulk.home('lib')

  require 'optparse'
  op = OptionParser.new

  op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
    # only for help message. implemented at lib/embulk/command/embulk.rb
  end

  case subcmd.to_sym
  when :bundle
    if default_bundle_path
      op.banner = "Usage: bundle [directory=#{default_bundle_path}]"
      args = 0..1
    else
      op.banner = "Usage: bundle <directory>"
      args = 1..1
    end

  when :run
    op.banner = "Usage: run <config.yml>"
    op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
    end
    op.on('-I', '--load-path PATH', 'Add ruby script directory path or jar file path') do |load_path|
      load_paths << load_path
    end
    op.on('-o', '--output PATH', 'Path to a file to write the next configuration') do |path|
      options[:nextConfigOutputPath] = path
    end
    args = 1..1

  when :preview
    op.banner = "Usage: preview <config.yml>"
    op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
    end
    op.on('-I', '--load-path PATH', 'Add ruby script directory path or jar file path') do |load_path|
      load_paths << load_path
    end
    args = 1..1

  when :guess
    op.banner = "Usage: guess <partial-config.yml>"
    op.on('-o', '--output PATH', 'Path to a file to write the guessed configuration') do |path|
      options[:nextConfigOutputPath] = path
    end
    op.on('-I', '--load-path PATH', 'Add ruby script directory path or jar file path') do |load_path|
      load_paths << load_path
    end
    args = 1..1

  #when :generate  # or :new
    # TODO create plugin templates

  when :gem
    require 'rubygems/gem_runner'
    Gem::GemRunner.new.run argv
    exit 0

  when :example
    args = 0..1

  when :exec
    exec *argv
    exit 127

  else
    usage "Unknown subcommand #{subcmd.dump}."
  end

  begin
    op.parse!(argv)
    unless args.include?(argv.length)
      usage nil
    end
  rescue => e
    usage e.to_s
  end

  case subcmd.to_sym
  when :bundle
    path = argv[0] || default_bundle_path

    require 'fileutils'
    require 'rubygems/gem_runner'
    setup_load_paths(load_paths)

    unless File.exists?(path)
      puts "Initializing #{path}..."
      FileUtils.mkdir_p File.dirname(path)
      begin
        success = false

        # copy embulk/data/bundle/ directory
        if __FILE__ =~ /^classpath:/ || __FILE__.include?('!/')
          # data is in embulk-core jar
          resource_class = org.embulk.command.Runner.java_class
          %w[.bundle/config embulk/input_example.rb embulk/output_example.rb Gemfile Gemfile.lock].each do |file|  # TODO get file list from the jar
            url = resource_class.resource("/embulk/data/bundle/#{file}").to_s
            dst = File.join(path, file)
            FileUtils.mkdir_p File.dirname(dst)
            FileUtils.cp(url, dst)
          end
        else
          #tmpl = File.join(File.dirname(__FILE__), '../data/bundle')
          tmpl = File.join(Embulk.home('lib'), 'embulk', 'data', 'bundle')
          FileUtils.cp_r tmpl, path
        end

        ## TODO this is disabled for now. enable this if you want to use
        ## create bin/embulk
        #bin_embulk_path = File.join(path, 'bin', 'embulk')
        #FileUtils.mkdir_p File.dirname(bin_embulk_path)
        #require 'embulk/command/embulk_generate_bin'  # defines Embulk.generate_bin
        #File.open(bin_embulk_path, 'wb', 0755) {|f| f.write Embulk.generate_bin(bundle_path: :here) }

        # install bundler
        setup_gem_paths(path)
        Gem::GemRunner.new.run %w[install bundler]

        success = true
      rescue Gem::SystemExitException => e
        raise e if e.exit_code != 0
        success = true
      ensure
        FileUtils.rm_rf path unless success
      end
    else
      setup_gem_paths(path)
    end

    ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(path, "Gemfile")
    Dir.chdir(path) do
      require 'bundler'
      require 'bundler/friendly_errors'
      require 'bundler/cli'
      Bundler.with_friendly_errors do
        # run > bundle install
        Bundler::CLI.start(%w[install], debug: true)
      end
    end

  when :example
    require_relative 'embulk_example'
    path = ARGV[0] || "embulk-example"
    puts "Creating #{path} directory..."
    Embulk.create_example(path)
    puts ""
    puts "Run following subcommands to try embulk:"
    puts ""
    puts "   1. guess #{File.join(path, 'example.yml')} -o config.yml"
    puts "   2. preview config.yml"
    puts "   3. run config.yml"
    puts ""

  else
    require 'json'

    begin
      java.lang.Class.forName('org.embulk.command.Runner')
    rescue java.lang.ClassNotFoundException
      # load classpath
      classpath_dir = Embulk.home('classpath')
      jars = Dir.entries(classpath_dir).select {|f| f =~ /\.jar$/ }.sort
      jars.each do |jar|
        require File.join(classpath_dir, jar)
      end
    end

    setup_load_paths(load_paths)

    org.embulk.command.Runner.new(options.to_json).main(subcmd, argv.to_java(:string))
  end
end

.setup_gem_paths(path) ⇒ Object 



206
207
208
209
210
211
 
# File 'lib/embulk/command/embulk_run.rb', line 206

def self.setup_gem_paths(path)
  # install bundler gem here & use bundler installed here
  ENV['GEM_HOME'] = File.expand_path File.join(path, Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
  ENV['GEM_PATH'] = ''
  Gem.clear_paths  # force rubygems to reload GEM_HOME
end

.setup_load_paths(load_paths) ⇒ Object 



213
214
215
216
217
218
219
220
221
222
223
 
# File 'lib/embulk/command/embulk_run.rb', line 213

def self.setup_load_paths(load_paths)
  load_paths.each do |load_path|
    if File.file?(load_path)
      # jar files
      require File.expand_path(load_path)
    else
      # ruby script directory (add at the beginning of $LOAD_PATH to make it highest priority)
      $LOAD_PATH.unshift File.expand_path(load_path)
    end
  end
end

.usage(message) ⇒ Object 



225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
 
# File 'lib/embulk/command/embulk_run.rb', line 225

def self.usage(message)
  STDERR.puts "usage: <command> [--options]"
  STDERR.puts "commands:"
  STDERR.puts "   bundle    [directory]                              # create or update plugin environment."
  STDERR.puts "   run       <config.yml>                             # run a bulk load transaction."
  STDERR.puts "   preview   <config.yml>                             # dry-run the bulk load without output and show preview."
  STDERR.puts "   guess     <partial-config.yml> -o <output.yml>     # guess missing parameters to create a complete configuration file."
  STDERR.puts "   gem       <install | list | help>                  # install a plugin or show installed plugins."
  STDERR.puts "                                                      # plugin path is #{ENV['GEM_HOME']}"
  STDERR.puts "   example   [path]                                   # creates an example config file and csv file to try embulk."
  STDERR.puts ""
  if message
    STDERR.puts "error: #{message}"
  else
    STDERR.puts "Use \`<command> --help\` to see description of the commands."
  end
  exit 1
end