Class: Bosh::Director::Jobs::UpdateRelease
- Includes:
- DownloadHelper, LockHelper
- Defined in:
- lib/bosh/director/jobs/update_release.rb
Instance Attribute Summary collapse
-
#release_model ⇒ Object
Returns the value of attribute release_model.
Attributes inherited from BaseJob
Class Method Summary collapse
Instance Method Summary collapse
-
#backfill_source_for_packages(packages, release_dir) ⇒ boolean
True if sources were added to at least one package; false if the call had no effect.
- #compiled_release ⇒ Object
- #create_compiled_package(package, stemcell, release_dir) ⇒ Object
-
#create_compiled_packages(all_compiled_packages, release_dir) ⇒ boolean
True if at least one job was created; false if the call had no effect.
- #create_job(job_meta, release_dir) ⇒ Object
-
#create_jobs(jobs, release_dir) ⇒ boolean
True if at least one job was created; false if the call had no effect.
-
#create_package(package_meta, release_dir) ⇒ void
Creates package in DB according to given metadata.
-
#create_packages(package_metas, release_dir) ⇒ Array<Hash>, boolean
Creates packages using provided metadata flag indicating if any changes were made to the database.
- #download_remote_release ⇒ Object
-
#extract_release ⇒ void
Extracts release tarball.
-
#initialize(release_path, options = {}) ⇒ UpdateRelease
constructor
A new instance of UpdateRelease.
-
#normalize_manifest ⇒ void
Normalizes release manifest, so all names, versions, and checksums are Strings.
-
#perform ⇒ void
Extracts release tarball, verifies release manifest and saves release in DB.
-
#process_jobs(release_dir) ⇒ void
Finds job template definitions in release manifest and sorts them into two buckets: new and existing job templates, then creates new job template records in the database and points release version to existing ones.
-
#process_packages(release_dir) ⇒ void
Finds all package definitions in the manifest and sorts them into two buckets: new and existing packages, then creates new packages and points current release version to the existing packages.
-
#process_release(release_dir) ⇒ void
Processes uploaded release, creates jobs and packages in DB if needed.
-
#register_package(package) ⇒ void
Marks package model as used by release version model.
-
#register_template(template) ⇒ void
Marks job template model as being used by release version.
-
#resolve_package_dependencies(packages) ⇒ void
Resolves package dependencies, makes sure there are no cycles and all dependencies are present.
-
#save_package_source_blob(package, package_meta, release_dir) ⇒ boolean
True if a new blob was created; false otherwise.
- #source_release ⇒ Object
- #stemcells_used_by_package(package_meta) ⇒ Object
-
#use_existing_jobs(jobs) ⇒ boolean
True if at least one job was tied to the release version; false if the call had no effect.
-
#use_existing_packages(packages, release_dir) ⇒ Array<Hash>
Points release DB model to existing packages described by given metadata.
- #validate_tgz(tgz, desc) ⇒ Object
- #verify_manifest(release_dir) ⇒ void
Methods included from DownloadHelper
Methods included from LockHelper
#with_compile_lock, #with_deployment_lock, #with_release_lock, #with_release_locks, #with_stemcell_lock
Methods inherited from BaseJob
#begin_stage, #event_log, #logger, perform, #result_file, #single_step_stage, #task_cancelled?, #task_checkpoint, #track_and_log
Constructor Details
#initialize(release_path, options = {}) ⇒ UpdateRelease
Returns a new instance of UpdateRelease.
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 |
# File 'lib/bosh/director/jobs/update_release.rb', line 21 def initialize(release_path, = {}) if ['remote'] # file will be downloaded to the release_path @release_path = File.join(Dir.tmpdir, "release-#{SecureRandom.uuid}") @release_url = release_path else # file already exists at the release_path @release_path = release_path end @release_model = nil @release_version_model = nil @rebase = !!['rebase'] @manifest = nil @name = nil @version = nil end |
Instance Attribute Details
#release_model ⇒ Object
Returns the value of attribute release_model.
13 14 15 |
# File 'lib/bosh/director/jobs/update_release.rb', line 13 def release_model @release_model end |
Class Method Details
.job_type ⇒ Object
15 16 17 |
# File 'lib/bosh/director/jobs/update_release.rb', line 15 def self.job_type :update_release end |
Instance Method Details
#backfill_source_for_packages(packages, release_dir) ⇒ boolean
Returns true if sources were added to at least one package; false if the call had no effect.
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 |
# File 'lib/bosh/director/jobs/update_release.rb', line 275 def backfill_source_for_packages(packages, release_dir) return false if packages.empty? had_effect = false single_step_stage("Processing #{packages.size} existing package#{"s" if packages.size > 1}") do packages.each do |package, | package_desc = "#{package.name}/#{package.version}" logger.info("Adding source for package `#{package_desc}'") had_effect |= save_package_source_blob(package, , release_dir) package.save end end had_effect end |
#compiled_release ⇒ Object
117 118 119 120 |
# File 'lib/bosh/director/jobs/update_release.rb', line 117 def compiled_release raise "Don't know what kind of release we have until verify_release is called" unless @manifest @compiled_release end |
#create_compiled_package(package, stemcell, release_dir) ⇒ Object
406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 |
# File 'lib/bosh/director/jobs/update_release.rb', line 406 def create_compiled_package(package, stemcell, release_dir) tgz = File.join(release_dir, 'compiled_packages', "#{package.name}.tgz") validate_tgz(tgz, "#{package.name}.tgz") compiled_package = Models::CompiledPackage.new compiled_package.blobstore_id = BlobUtil.create_blob(tgz) compiled_package.sha1 = Digest::SHA1.file(tgz).hexdigest transitive_dependencies = @release_version_model.transitive_dependencies(package) compiled_package.dependency_key = Models::CompiledPackage.create_dependency_key(transitive_dependencies) compiled_package.build = Models::CompiledPackage.generate_build_number(package, stemcell) compiled_package.package_id = package.id compiled_package.stemcell_id = stemcell.id compiled_package.save end |
#create_compiled_packages(all_compiled_packages, release_dir) ⇒ boolean
Returns true if at least one job was created; false if the call had no effect.
361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 |
# File 'lib/bosh/director/jobs/update_release.rb', line 361 def create_compiled_packages(all_compiled_packages, release_dir) return false if all_compiled_packages.nil? event_log.begin_stage('Creating new compiled packages', all_compiled_packages.size) had_effect = false all_compiled_packages.each do |compiled_package_spec| package = compiled_package_spec[:package] stemcell = compiled_package_spec[:stemcell] existing_compiled_package = Models::CompiledPackage.where( :package_id => package.id, :stemcell_id => stemcell.id) if existing_compiled_package.empty? package_desc = "#{package.name}/#{package.version} for #{stemcell.name}/#{stemcell.version}" event_log.track(package_desc) do create_compiled_package(package, stemcell, release_dir) had_effect = true end end end had_effect end |
#create_job(job_meta, release_dir) ⇒ Object
541 542 543 544 |
# File 'lib/bosh/director/jobs/update_release.rb', line 541 def create_job(, release_dir) release_job = ReleaseJob.new(, @release_model, release_dir, @packages, logger) release_job.create end |
#create_jobs(jobs, release_dir) ⇒ boolean
Returns true if at least one job was created; false if the call had no effect.
525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 |
# File 'lib/bosh/director/jobs/update_release.rb', line 525 def create_jobs(jobs, release_dir) return false if jobs.empty? event_log.begin_stage("Creating new jobs", jobs.size) jobs.each do || job_desc = "#{["name"]}/#{["version"]}" event_log.track(job_desc) do logger.info("Creating new template `#{job_desc}'") template = create_job(, release_dir) register_template(template) end end true end |
#create_package(package_meta, release_dir) ⇒ void
This method returns an undefined value.
Creates package in DB according to given metadata
429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 |
# File 'lib/bosh/director/jobs/update_release.rb', line 429 def create_package(, release_dir) name, version = ['name'], ['version'] package_attrs = { :release => @release_model, :name => name, :sha1 => nil, :blobstore_id => nil, :fingerprint => ['fingerprint'], :version => version } package = Models::Package.new(package_attrs) package.dependency_set = ['dependencies'] save_package_source_blob(package, , release_dir) unless @compiled_release package.save end |
#create_packages(package_metas, release_dir) ⇒ Array<Hash>, boolean
Creates packages using provided metadata flag indicating if any changes were made to the database.
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 |
# File 'lib/bosh/director/jobs/update_release.rb', line 330 def create_packages(, release_dir) if .empty? return [], false end package_stemcell_hashes = [] event_log.begin_stage("Creating new packages", .size) .each do || package_desc = "#{["name"]}/#{["version"]}" package = nil event_log.track(package_desc) do logger.info("Creating new package `#{package_desc}'") package = create_package(, release_dir) register_package(package) end if @compiled_release stemcells = stemcells_used_by_package() stemcells.each do |stemcell| hash = { package: package, stemcell: stemcell} package_stemcell_hashes << hash end end end return package_stemcell_hashes, true end |
#download_remote_release ⇒ Object
69 70 71 |
# File 'lib/bosh/director/jobs/update_release.rb', line 69 def download_remote_release download_remote_file('release', @release_url, @release_path) end |
#extract_release ⇒ void
This method returns an undefined value.
Extracts release tarball
75 76 77 78 79 80 81 82 83 84 85 86 |
# File 'lib/bosh/director/jobs/update_release.rb', line 75 def extract_release release_dir = Dir.mktmpdir result = Bosh::Exec.sh("tar -C #{release_dir} -xzf #{@release_path} 2>&1", :on_error => :return) if result.failed? logger.error("Failed to extract release archive '#{@release_path}' into dir '#{release_dir}', tar returned #{result.exit_status}, output: #{result.output})") FileUtils.rm_rf(release_dir) raise ReleaseInvalidArchive, "Extracting release archive failed. Check task debug log for details." end release_dir end |
#normalize_manifest ⇒ void
This method returns an undefined value.
Normalizes release manifest, so all names, versions, and checksums are Strings.
159 160 161 162 163 164 |
# File 'lib/bosh/director/jobs/update_release.rb', line 159 def normalize_manifest Bosh::Director.hash_string_vals(@manifest, 'name', 'version') @manifest[@packages_folder].each { |p| Bosh::Director.hash_string_vals(p, 'name', 'version', 'sha1') } @manifest['jobs'].each { |j| Bosh::Director.hash_string_vals(j, 'name', 'version', 'sha1') } end |
#perform ⇒ void
This method returns an undefined value.
Extracts release tarball, verifies release manifest and saves release in DB
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 |
# File 'lib/bosh/director/jobs/update_release.rb', line 43 def perform logger.info("Processing update release") logger.info("Release rebase will be performed") if @rebase single_step_stage("Downloading remote release") { download_remote_release } if @release_url release_dir = nil single_step_stage("Extracting release") { release_dir = extract_release } single_step_stage("Verifying manifest") { verify_manifest(release_dir) } with_release_lock(@name) { process_release(release_dir) } "Created release `#{@name}/#{@version}'" rescue Exception => e remove_release_version_model raise e ensure FileUtils.rm_rf(release_dir) if release_dir FileUtils.rm_rf(@release_path) if @release_path end |
#process_jobs(release_dir) ⇒ void
This method returns an undefined value.
Finds job template definitions in release manifest and sorts them into two buckets: new and existing job templates, then creates new job template records in the database and points release version to existing ones.
495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 |
# File 'lib/bosh/director/jobs/update_release.rb', line 495 def process_jobs(release_dir) logger.info("Checking for new jobs in release") new_jobs = [] existing_jobs = [] @manifest["jobs"].each do || # Checking whether we might have the same bits somewhere jobs = Models::Template.where(fingerprint: ["fingerprint"]).all template = jobs.find do |job| job.release_id == @release_model.id && job.name == ["name"] && job.version == ["version"] end if template.nil? new_jobs << else existing_jobs << [template, ] end end did_something = create_jobs(new_jobs, release_dir) did_something |= use_existing_jobs(existing_jobs) did_something end |
#process_packages(release_dir) ⇒ void
This method returns an undefined value.
Finds all package definitions in the manifest and sorts them into two buckets: new and existing packages, then creates new packages and points current release version to the existing packages.
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 |
# File 'lib/bosh/director/jobs/update_release.rb', line 195 def process_packages(release_dir) logger.info("Checking for new packages in release") new_packages = [] existing_packages = [] registered_packages = [] @manifest[@packages_folder].each do || # Checking whether we might have the same bits somewhere packages = Models::Package.where(fingerprint: ["fingerprint"]).all if packages.empty? unless @release_is_new raise ReleaseInvalidPackage, "package #{['name']}/#{['version']} not part of previous upload of release #{@name}/#{@version}" end new_packages << next end existing_package = packages.find do |package| package.release_id == @release_model.id && package.name == ["name"] && package.version == ["version"] end if existing_package # clean up 'broken' dependency_set (a bug was including transitives) # dependency ordering impacts fingerprint # TODO: The following code can be removed after some reasonable time period (added 2014.10.06) if existing_package.dependency_set != ['dependencies'] existing_package.dependency_set = ['dependencies'] existing_package.save end if existing_package.release_versions.include? @release_version_model registered_packages << [existing_package, ] else existing_packages << [existing_package, ] end else # We found a package with the same fingerprint but different # (release, name, version) tuple, so we need to make a copy # of the package blob and create a new db entry for it packages.each do |package| unless package.blobstore_id.nil? ["blobstore_id"] = package.blobstore_id ["sha1"] = package.sha1 break end end new_packages << end end did_something = false package_stemcell_hashes1, created_package = create_packages(new_packages, release_dir) did_something |= created_package package_stemcell_hashes2, modified_package = use_existing_packages(existing_packages, release_dir) did_something |= modified_package if @compiled_release compatible_stemcell_combos = registered_packages.flat_map do |pkg, | stemcells_used_by_package().map do |stemcell| { package: pkg, stemcell: stemcell } end end consolidated_package_stemcell_hashes = Array(package_stemcell_hashes1) | Array(package_stemcell_hashes2) | compatible_stemcell_combos did_something |= create_compiled_packages(consolidated_package_stemcell_hashes, release_dir) else did_something |= backfill_source_for_packages(registered_packages, release_dir) end did_something end |
#process_release(release_dir) ⇒ void
This method returns an undefined value.
Processes uploaded release, creates jobs and packages in DB if needed
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 |
# File 'lib/bosh/director/jobs/update_release.rb', line 129 def process_release(release_dir) @release_model = Models::Release.find_or_create(:name => @name) if @rebase @version = next_release_version end version_attrs = { :release => @release_model, :version => @version.to_s } version_attrs[:uncommitted_changes] = @uncommitted_changes if @uncommitted_changes version_attrs[:commit_hash] = @commit_hash if @commit_hash @release_is_new = false @release_version_model = Models::ReleaseVersion.find_or_create(version_attrs) { @release_is_new = true } single_step_stage("Resolving package dependencies") do resolve_package_dependencies(@manifest[@packages_folder]) end @packages = {} process_packages(release_dir) process_jobs(release_dir) if @release_is_new event_log.begin_stage(@compiled_release ? "Compiled Release has been created" : "Release has been created", 1) event_log.track("#{@name}/#{@version}") {} end |
#register_package(package) ⇒ void
This method returns an undefined value.
Marks package model as used by release version model
485 486 487 488 |
# File 'lib/bosh/director/jobs/update_release.rb', line 485 def register_package(package) @packages[package.name] = package @release_version_model.add_package(package) end |
#register_template(template) ⇒ void
This method returns an undefined value.
Marks job template model as being used by release version
565 566 567 |
# File 'lib/bosh/director/jobs/update_release.rb', line 565 def register_template(template) @release_version_model.add_template(template) end |
#resolve_package_dependencies(packages) ⇒ void
This method returns an undefined value.
Resolves package dependencies, makes sure there are no cycles and all dependencies are present
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 |
# File 'lib/bosh/director/jobs/update_release.rb', line 169 def resolve_package_dependencies(packages) packages_by_name = {} packages.each do |package| packages_by_name[package["name"]] = package package["dependencies"] ||= [] end logger.info("Resolving package dependencies for #{packages_by_name.keys.inspect}") dependency_lookup = lambda do |package_name| packages_by_name[package_name]["dependencies"] end result = Bosh::Director::CycleHelper.check_for_cycle(packages_by_name.keys, :connected_vertices => true, &dependency_lookup) packages.each do |package| name = package["name"] dependencies = package["dependencies"] all_dependencies = result[:connected_vertices][name] logger.info("Resolved package dependencies for `#{name}': #{dependencies.pretty_inspect} => #{all_dependencies.pretty_inspect}") end end |
#save_package_source_blob(package, package_meta, release_dir) ⇒ boolean
Returns true if a new blob was created; false otherwise.
450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 |
# File 'lib/bosh/director/jobs/update_release.rb', line 450 def save_package_source_blob(package, , release_dir) return false unless package.blobstore_id.nil? name, version = ['name'], ['version'] existing_blob = ['blobstore_id'] desc = "package '#{name}/#{version}'" package.sha1 = ['sha1'] if existing_blob logger.info("Creating #{desc} from existing blob #{existing_blob}") package.blobstore_id = BlobUtil.copy_blob(existing_blob) elsif package logger.info("Creating #{desc} from provided bits") package_tgz = File.join(release_dir, 'packages', "#{name}.tgz") validate_tgz(package_tgz, desc) package.blobstore_id = BlobUtil.create_blob(package_tgz) end true end |
#source_release ⇒ Object
122 123 124 |
# File 'lib/bosh/director/jobs/update_release.rb', line 122 def source_release !compiled_release end |
#stemcells_used_by_package(package_meta) ⇒ Object
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 |
# File 'lib/bosh/director/jobs/update_release.rb', line 386 def stemcells_used_by_package() if ['stemcell'].nil? raise 'stemcell informatiom(operating system/version) should be listed for each package of a compiled tarball' end values = ['stemcell'].split('/', 2) = values[0] stemcell_version = values[1] unless && stemcell_version raise 'stemcell informatiom(operating system/version) should be listed for each package of a compiled tarball' end stemcells = Models::Stemcell.where(:operating_system => , :version => stemcell_version) if stemcells.empty? raise "No stemcells matching OS #{} version #{stemcell_version}" end stemcells end |
#use_existing_jobs(jobs) ⇒ boolean
Returns true if at least one job was tied to the release version; false if the call had no effect.
548 549 550 551 552 553 554 555 556 557 558 559 560 |
# File 'lib/bosh/director/jobs/update_release.rb', line 548 def use_existing_jobs(jobs) return false if jobs.empty? single_step_stage("Processing #{jobs.size} existing job#{"s" if jobs.size > 1}") do jobs.each do |template, _| job_desc = "#{template.name}/#{template.version}" logger.info("Using existing job `#{job_desc}'") register_template(template) end end true end |
#use_existing_packages(packages, release_dir) ⇒ Array<Hash>
Points release DB model to existing packages described by given metadata
294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 |
# File 'lib/bosh/director/jobs/update_release.rb', line 294 def use_existing_packages(packages, release_dir) if packages.empty? return [], false end package_stemcell_hashes = [] single_step_stage("Processing #{packages.size} existing package#{"s" if packages.size > 1}") do packages.each do |package, | package_desc = "#{package.name}/#{package.version}" logger.info("Using existing package `#{package_desc}'") register_package(package) if compiled_release stemcells = stemcells_used_by_package() stemcells.each do |stemcell| hash = { package: package, stemcell: stemcell} package_stemcell_hashes << hash end end if source_release && package.blobstore_id.nil? save_package_source_blob(package, , release_dir) package.save end end end return package_stemcell_hashes, true end |
#validate_tgz(tgz, desc) ⇒ Object
474 475 476 477 478 479 480 |
# File 'lib/bosh/director/jobs/update_release.rb', line 474 def validate_tgz(tgz, desc) result = Bosh::Exec.sh("tar -tzf #{tgz} 2>&1", :on_error => :return) if result.failed? logger.error("Extracting #{desc} archive failed, tar returned #{result.exit_status}, output: #{result.output}") raise PackageInvalidArchive, "Extracting #{desc} archive failed. Check task debug log for details." end end |
#verify_manifest(release_dir) ⇒ void
This method returns an undefined value.
90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 |
# File 'lib/bosh/director/jobs/update_release.rb', line 90 def verify_manifest(release_dir) manifest_file = File.join(release_dir, "release.MF") raise ReleaseManifestNotFound, "Release manifest not found" unless File.file?(manifest_file) @manifest = Psych.load_file(manifest_file) #handle compiled_release case @compiled_release = !!@manifest["compiled_packages"] @packages_folder = @compiled_release ? "compiled_packages" : "packages" normalize_manifest @name = @manifest["name"] begin @version = Bosh::Common::Version::ReleaseVersion.parse(@manifest["version"]) unless @version == @manifest["version"] logger.info("Formatted version '#{@manifest["version"]}' => '#{@version}'") end rescue SemiSemantic::ParseError raise ReleaseVersionInvalid, "Release version invalid: #{@manifest["version"]}" end @commit_hash = @manifest.fetch("commit_hash", nil) @uncommitted_changes = @manifest.fetch("uncommitted_changes", nil) end |