Class: Bosh::Agent::Message::CompilePackage

Inherits:
Object
  • Object
show all
Defined in:
lib/bosh_agent/message/compile_package.rb

Overview

This agent message “compile_package” fetches the source package from the blobstore, fetches the dependency compiled packages, compiles the source package, packs it into a tgz blob, and uploads it to the same blobstore. It returns the uploaded blob’s blobstore_id & sha1.

This message has the following uses:

  • the bosh-release (within the stemcell_builder) to compile packages to the microbosh’s local blobstore

  • the director requests packages be compiled during deployment if they are not yet compiled and available in the blobstore

Source packages must have a ‘packaging` executable script. It can find the unpackaged source files in the directory it is run in (which is also provided as $BOSH_COMPILE_TARGET variable). The packaging script MUST place all compiled assets within the single folder specified as $BOSH_INSTALL_TARGET. The packaging script is also provided the environment variables $BOSH_PACKAGE_NAME and $BOSH_PACKAGE_VERSION

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(args) ⇒ CompilePackage

Returns a new instance of CompilePackage.



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/bosh_agent/message/compile_package.rb', line 35

def initialize(args)
  bsc_provider = Bosh::Agent::Config.blobstore_provider
  bsc_options = Bosh::Agent::Config.blobstore_options
  @blobstore_client = Bosh::Blobstore::Client.safe_create(bsc_provider, bsc_options)
  @blobstore_id, @sha1, @package_name, @package_version, @dependencies = args

  @base_dir = Bosh::Agent::Config.base_dir

  # The maximum amount of disk percentage that can be used during
  # compilation before an error will be thrown.  This is to prevent
  # package compilation throwing arbitrary errors when disk space runs
  # out.
  # @attr [Integer] The max percentage of disk that can be used in
  #     compilation.
  @max_disk_usage_pct = 90
  FileUtils.mkdir_p(File.join(@base_dir, 'data', 'tmp'))

  @log_file = "#{@base_dir}/data/tmp/#{Bosh::Agent::Config.agent_id}"
  @logger = Logger.new(@log_file)
  @logger.level = Logger::DEBUG
  @compile_base = "#{@base_dir}/data/compile"
  @install_base = "#{@base_dir}/data/packages"
end

Instance Attribute Details

#blobstore_client ⇒ Object (readonly)

Returns the value of attribute blobstore_client.



28
29
30
 
# File 'lib/bosh_agent/message/compile_package.rb', line 28

def blobstore_client
  @blobstore_client
end

#blobstore_id ⇒ Object

Returns the value of attribute blobstore_id.



26
27
28
 
# File 'lib/bosh_agent/message/compile_package.rb', line 26

def blobstore_id
  @blobstore_id
end

#compile_base ⇒ Object

Returns the value of attribute compile_base.



27
28
29
 
# File 'lib/bosh_agent/message/compile_package.rb', line 27

def compile_base
  @compile_base
end

#install_base ⇒ Object

Returns the value of attribute install_base.



27
28
29
 
# File 'lib/bosh_agent/message/compile_package.rb', line 27

def install_base
  @install_base
end

#package_name ⇒ Object

Returns the value of attribute package_name.



26
27
28
 
# File 'lib/bosh_agent/message/compile_package.rb', line 26

def package_name
  @package_name
end

#package_sha1 ⇒ Object

Returns the value of attribute package_sha1.



26
27
28
 
# File 'lib/bosh_agent/message/compile_package.rb', line 26

def package_sha1
  @package_sha1
end

#package_version ⇒ Object

Returns the value of attribute package_version.



26
27
28
 
# File 'lib/bosh_agent/message/compile_package.rb', line 26

def package_version
  @package_version
end

Class Method Details

.long_running? ⇒ Boolean

Returns:

  • (Boolean) 


33
 
# File 'lib/bosh_agent/message/compile_package.rb', line 33

def self.long_running?; true; end

.process(args) ⇒ Object 



30
31
32
 
# File 'lib/bosh_agent/message/compile_package.rb', line 30

def self.process(args)
  self.new(args).start
end

Instance Method Details

#clear_log_file(log_file) ⇒ Object

Clears the log file after a compilation runs. This is needed because if reuse_compilation_vms is being used then without clearing the log then the log from each subsequent compilation will include the previous compilation’s output.

Parameters:

  • log_file (String) —

    Path to the log file.



228
229
230
231
 
# File 'lib/bosh_agent/message/compile_package.rb', line 228

def clear_log_file(log_file)
  File.delete(log_file) if File.exists?(log_file)
  @logger = Logger.new(log_file)
end

#compile ⇒ Object 



177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/bosh_agent/message/compile_package.rb', line 177

def compile
  FileUtils.rm_rf install_dir if File.directory?(install_dir)
  FileUtils.mkdir_p install_dir

  pkg_link_dst = File.join(@base_dir, 'packages', @package_name)
  FileUtils.ln_sf(install_dir, pkg_link_dst)

  pct_space_used = pct_disk_used(@compile_base)
  if pct_space_used >= @max_disk_usage_pct
    raise Bosh::Agent::MessageHandlerError,
        "Compile Package Failure. Greater than #{@max_disk_usage_pct}% " +
        "is used (#{pct_space_used}%."
  end
  Dir.chdir(compile_dir) do

    # Prevent these from getting inhereted from the agent
    %w{GEM_HOME BUNDLE_GEMFILE RUBYOPT}.each { |key| ENV.delete(key) }

    ENV['BOSH_COMPILE_TARGET'] = compile_dir
    ENV['BOSH_INSTALL_TARGET'] = pkg_link_dst
    ENV['BOSH_PACKAGE_NAME'] = @package_name.to_s
    ENV['BOSH_PACKAGE_VERSION'] = @package_version.to_s
    if File.exist?('packaging')
      @logger.info("Compiling #{@package_name} #{@package_version}")
      output = `bash -x packaging 2>&1`
      # stick the output in the blobstore
      @logger.fatal(output)
      unless $?.exitstatus == 0
        raise Bosh::Agent::MessageHandlerError.new(
          "Compile Package Failure (exit code: #{$?.exitstatus})", output)
      end
    end
  end
end

#compile_dir ⇒ Object 



115
116
117
 
# File 'lib/bosh_agent/message/compile_package.rb', line 115

def compile_dir
  @compile_dir ||= File.join(@compile_base, @package_name)
end

#compiled_package ⇒ Object 



212
213
214
 
# File 'lib/bosh_agent/message/compile_package.rb', line 212

def compiled_package
  File.join(@source_file + ".compiled")
end

#delete_tmp_files ⇒ Object

Delete the leftover compilation files after a compilation is done. This is done so that the reuse_compilation_vms option does not fill up a VM.



80
81
82
83
84
85
86
 
# File 'lib/bosh_agent/message/compile_package.rb', line 80

def delete_tmp_files
  [@compile_base, @install_base].each do |dir|
    if Dir.exists?(dir)
      FileUtils.rm_rf(dir)
    end
  end
end

#disk_info_as_array(path) ⇒ Object 



140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/bosh_agent/message/compile_package.rb', line 140

def disk_info_as_array(path)
  # "Filesystem   1024-blocks     Used Available Capacity  Mounted on\n
  # /dev/disk0s2   195312496 92743676 102312820    48%    /\n"
  df_out = `df -Pk #{path} 2>&1`
  unless $?.exitstatus == 0
    raise Bosh::Agent::MessageHandlerError, "Command 'df -Pk #{path}' " +
        "on compilation VM failed with:\n#{df_out}\nexit code: " +
        "#{$?.exitstatus}"
  end

  # "/dev/disk0s2   195312496 92743568 102312928    48%    /\n"
  df_out = df_out.sub(/[^\/]*/, "")
  # ["/dev/disk0s2", "195312496", "92743568", "102312928", "48%", "/\n"]
  df_out.split(/[ ]+/)
end

#disk_total(path) ⇒ Integer

Get the amount of total disk (in KBytes).

Parameters:

  • path (String) —

    Path that the disk size is being requested for.

Returns:

  • (Integer) —

    The total disk size (in KBytes).



159
160
161
 
# File 'lib/bosh_agent/message/compile_package.rb', line 159

def disk_total(path)
  disk_info_as_array(path)[1].to_i
end

#disk_used(path) ⇒ Integer

Get the amount of disk being used (in KBytes).

Parameters:

  • path (String) —

    Path that the disk usage is being requested for.

Returns:

  • (Integer) —

    The amount being used (in KBytes).



166
167
168
 
# File 'lib/bosh_agent/message/compile_package.rb', line 166

def disk_used(path)
  disk_info_as_array(path)[2].to_i
end

#get_source_package ⇒ Object 



104
105
106
107
108
109
110
111
112
113
 
# File 'lib/bosh_agent/message/compile_package.rb', line 104

def get_source_package
  compile_tmp = File.join(@compile_base, 'tmp')
  FileUtils.mkdir_p compile_tmp
  @source_file = File.join(compile_tmp, @blobstore_id)
  FileUtils.rm @source_file if File.exist?(@source_file)

  File.open(@source_file, 'w') do |f|
    @blobstore_client.get(@blobstore_id, f)
  end
end

#install_dependencies ⇒ Object 



88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/bosh_agent/message/compile_package.rb', line 88

def install_dependencies
  @logger.info("Installing Dependencies")
  @dependencies.each do |pkg_name, pkg|
    @logger.info("Installing depdendency: #{pkg_name} #{pkg.inspect}")

    blobstore_id = pkg['blobstore_id']
    sha1 = pkg['sha1']
    install_dir = File.join(@install_base, pkg_name, pkg['version'])

    Util.unpack_blob(blobstore_id, sha1, install_dir)

    pkg_link_dst = File.join(@base_dir, 'packages', pkg_name)
    FileUtils.ln_sf(install_dir, pkg_link_dst)
  end
end

#install_dir ⇒ Object 



119
120
121
 
# File 'lib/bosh_agent/message/compile_package.rb', line 119

def install_dir
  @install_dir ||= File.join(@install_base, @package_name, @package_version.to_s)
end

#pack ⇒ Object 



216
217
218
219
220
221
 
# File 'lib/bosh_agent/message/compile_package.rb', line 216

def pack
  @logger.info("Packing #{@package_name} #{@package_version}")
  Dir.chdir(install_dir) do
    `tar -zcf #{compiled_package} .`
  end
end

#pct_disk_used(path) ⇒ Float

Get the percentage of disk that is used on the compilation VM.

Parameters:

  • path (String) —

    Path that the disk usage is being requested for.

Returns:

  • (Float) —

    The percentage of disk in use.



173
174
175
 
# File 'lib/bosh_agent/message/compile_package.rb', line 173

def pct_disk_used(path)
  100 * disk_used(path).to_f / disk_total(path).to_f
end

#start ⇒ Object 



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/bosh_agent/message/compile_package.rb', line 59

def start
  begin
    install_dependencies
    get_source_package
    unpack_source_package
    compile
    pack
    result = upload
    clear_log_file(@log_file)
    return { "result" => result }
  rescue RuntimeError => e
    @logger.warn("%s\n%s" % [e.message, e.backtrace.join("\n")])
    raise Bosh::Agent::MessageHandlerError, e
  ensure
    delete_tmp_files
  end
end

#unpack_source_package ⇒ Object 



123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/bosh_agent/message/compile_package.rb', line 123

def unpack_source_package
  FileUtils.rm_rf compile_dir if File.directory?(compile_dir)

  FileUtils.mkdir_p compile_dir
  Dir.chdir(compile_dir) do
    output = `tar -zxf #{@source_file} 2>&1`
    @logger.info(output)
    # stick the output in the blobstore
    unless $?.exitstatus == 0
      STDOUT.puts(output)
      raise Bosh::Agent::MessageHandlerError.new(
        "Compile Package Unpack Source Failure (exit code: #{$?.exitstatus})",
        output)
    end
  end
end

#upload ⇒ Object 



233
234
235
236
237
238
239
240
241
242
243
244
245
246
 
# File 'lib/bosh_agent/message/compile_package.rb', line 233

def upload
  compiled_blobstore_id = nil
  File.open(compiled_package, 'r') do |f|
    compiled_blobstore_id = @blobstore_client.create(f)
  end
  compiled_sha1 = Digest::SHA1.file(compiled_package).hexdigest
  compile_log_id = @blobstore_client.create(@log_file)
  @logger.info("Uploaded #{@package_name} #{@package_version} " +
               "(sha1: #{compiled_sha1}, " +
               "blobstore_id: #{compiled_blobstore_id})")
  @logger = nil
  { "sha1" => compiled_sha1, "blobstore_id" => compiled_blobstore_id,
    "compile_log_id" => compile_log_id }
end