Class: Bosh::Director::Jobs::UpdateRelease

Inherits:
BaseJob show all
Includes:
DownloadHelper, LockHelper
Defined in:
lib/bosh/director/jobs/update_release.rb

Instance Attribute Summary collapse

Attributes inherited from BaseJob

#task_id

Class Method Summary collapse

Instance Method Summary collapse

Methods included from DownloadHelper

#download_remote_file

Methods included from LockHelper

#with_compile_lock, #with_deployment_lock, #with_release_lock, #with_release_locks, #with_stemcell_lock

Methods inherited from BaseJob

#begin_stage, #event_log, #logger, perform, #result_file, #single_step_stage, #task_cancelled?, #task_checkpoint, #track_and_log

Constructor Details

#initialize(tmp_release_dir, options = {}) ⇒ UpdateRelease

Returns a new instance of UpdateRelease.

Parameters:

  • tmp_release_dir (String) —

    Directory containing release bundle

  • options (Hash) (defaults to: {}) —

    Release update options



22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/bosh/director/jobs/update_release.rb', line 22

def initialize(tmp_release_dir, options = {})
  @tmp_release_dir = tmp_release_dir
  @release_model = nil
  @release_version_model = nil

  @rebase = !!options["rebase"]
  @package_rebase_mapping = {}
  @job_rebase_mapping = {}

  @manifest = nil
  @name = nil
  @version = nil

  @packages_unchanged = false
  @jobs_unchanged = false
  
  @remote_release = options['remote'] || false
  @remote_release_location = options['location'] if @remote_release 
end

Instance Attribute Details

#release_model ⇒ Object

Returns the value of attribute release_model.



13
14
15
 
# File 'lib/bosh/director/jobs/update_release.rb', line 13

def release_model
  @release_model
end

#tmp_release_dir ⇒ Object

Returns the value of attribute tmp_release_dir.



14
15
16
 
# File 'lib/bosh/director/jobs/update_release.rb', line 14

def tmp_release_dir
  @tmp_release_dir
end

Class Method Details

.job_type ⇒ Object 



16
17
18
 
# File 'lib/bosh/director/jobs/update_release.rb', line 16

def self.job_type
  :update_release
end

Instance Method Details

#create_job(job_meta) ⇒ Object 



456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
 
# File 'lib/bosh/director/jobs/update_release.rb', line 456

def create_job(job_meta)
  name, version = job_meta["name"], job_meta["version"]

  template_attrs = {
    :release => @release_model,
    :name => name,
    :sha1 => job_meta["sha1"],
    :fingerprint => job_meta["fingerprint"],
    :version => version
  }

  if @rebase
    new_version = next_template_version(name, version)
    if new_version != version
      transition = "#{version} -> #{new_version}"
      logger.info("Job `#{name}' rebased: #{transition}")
      template_attrs[:version] = new_version
      version = new_version
      @job_rebase_mapping[name] = transition
    end
  end

  logger.info("Creating job template `#{name}/#{version}' " +
              "from provided bits")
  template = Models::Template.new(template_attrs)

  job_tgz = File.join(@tmp_release_dir, "jobs", "#{name}.tgz")
  job_dir = File.join(@tmp_release_dir, "jobs", "#{name}")

  FileUtils.mkdir_p(job_dir)

  desc = "job `#{name}/#{version}'"
  result = Bosh::Exec.sh("tar -C #{job_dir} -xzf #{job_tgz} 2>&1", :on_error => :return)
  if result.failed?
    logger.error("Extracting #{desc} archive failed in dir #{job_dir}, " +
                 "tar returned #{result.exit_status}, " +
                 "output: #{result.output}")
    raise JobInvalidArchive, "Extracting #{desc} archive failed. Check task debug log for details."
  end

  manifest_file = File.join(job_dir, "job.MF")
  unless File.file?(manifest_file)
    raise JobMissingManifest,
          "Missing job manifest for `#{template.name}'"
  end

  job_manifest = Psych.load_file(manifest_file)

  if job_manifest["templates"]
    job_manifest["templates"].each_key do |relative_path|
      path = File.join(job_dir, "templates", relative_path)
      unless File.file?(path)
        raise JobMissingTemplateFile,
              "Missing template file `#{relative_path}' " +
              "for job `#{template.name}'"
      end
    end
  end

  main_monit_file = File.join(job_dir, "monit")
  aux_monit_files = Dir.glob(File.join(job_dir, "*.monit"))

  unless File.exists?(main_monit_file) || aux_monit_files.size > 0
    raise JobMissingMonit, "Job `#{template.name}' is missing monit file"
  end

  template.blobstore_id = BlobUtil.create_blob(job_tgz)

  package_names = []
  job_manifest["packages"].each do |package_name|
    package = @packages[package_name]
    if package.nil?
      raise JobMissingPackage,
            "Job `#{template.name}' is referencing " +
            "a missing package `#{package_name}'"
    end
    package_names << package.name
  end
  template.package_names = package_names

  if job_manifest["logs"]
    unless job_manifest["logs"].is_a?(Hash)
      raise JobInvalidLogSpec,
            "Job `#{template.name}' has invalid logs spec format"
    end

    template.logs = job_manifest["logs"]
  end

  if job_manifest["properties"]
    unless job_manifest["properties"].is_a?(Hash)
      raise JobInvalidPropertySpec,
            "Job `#{template.name}' has invalid properties spec format"
    end

    template.properties = job_manifest["properties"]
  end

  template.save
end

#create_jobs(jobs) ⇒ Object 



439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
 
# File 'lib/bosh/director/jobs/update_release.rb', line 439

def create_jobs(jobs)
  if jobs.empty?
    @jobs_unchanged = true
    return
  end

  event_log.begin_stage("Creating new jobs", jobs.size)
  jobs.each do |job_meta|
    job_desc = "#{job_meta["name"]}/#{job_meta["version"]}"
    event_log.track(job_desc) do
      logger.info("Creating new template `#{job_desc}'")
      template = create_job(job_meta)
      register_template(template)
    end
  end
end

#create_package(package_meta) ⇒ void

This method returns an undefined value.

Creates package in DB according to given metadata

Parameters:

  • package_meta (Hash) —

    Package metadata



325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
 
# File 'lib/bosh/director/jobs/update_release.rb', line 325

def create_package(package_meta)
  name, version = package_meta["name"], package_meta["version"]

  package_attrs = {
    :release => @release_model,
    :name => name,
    :sha1 => package_meta["sha1"],
    :fingerprint => package_meta["fingerprint"],
    :version => version
  }

  if @rebase
    new_version = next_package_version(name, version)
    if new_version != version
      transition = "#{version} -> #{new_version}"
      logger.info("Package `#{name}' rebased: #{transition}")
      package_attrs[:version] = new_version
      version = new_version
      @package_rebase_mapping[name] = transition
    end
  end

  package = Models::Package.new(package_attrs)
  package.dependency_set = package_meta["dependencies"]

  existing_blob = package_meta["blobstore_id"]
  desc = "package `#{name}/#{version}'"

  if existing_blob
    logger.info("Creating #{desc} from existing blob #{existing_blob}")
    package.blobstore_id = BlobUtil.copy_blob(existing_blob)
  else
    logger.info("Creating #{desc} from provided bits")

    package_tgz = File.join(@tmp_release_dir, "packages", "#{name}.tgz")
    result = Bosh::Exec.sh("tar -tzf #{package_tgz} 2>&1", :on_error => :return)
    if result.failed?
      logger.error("Extracting #{desc} archive failed, " +
                   "tar returned #{result.exit_status}, " +
                   "output: #{result.output}")
      raise PackageInvalidArchive, "Extracting #{desc} archive failed. Check task debug log for details."
    end

    package.blobstore_id = BlobUtil.create_blob(package_tgz)
  end

  package.save
end

#create_packages(packages) ⇒ void

This method returns an undefined value.

Creates packages using provided metadata

Parameters:

  • packages (Array<Hash>) —

    Packages metadata



276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
 
# File 'lib/bosh/director/jobs/update_release.rb', line 276

def create_packages(packages)
  if packages.empty?
    @packages_unchanged = true
    return
  end

  event_log.begin_stage("Creating new packages", packages.size)
  packages.each do |package_meta|
    package_desc = "#{package_meta["name"]}/#{package_meta["version"]}"
    event_log.track(package_desc) do
      logger.info("Creating new package `#{package_desc}'")
      package = create_package(package_meta)
      register_package(package)
    end
  end
end

#download_remote_release ⇒ Object 



72
73
74
75
 
# File 'lib/bosh/director/jobs/update_release.rb', line 72

def download_remote_release         
  release_file = File.join(@tmp_release_dir, Api::ReleaseManager::RELEASE_TGZ)
  download_remote_file('release', @remote_release_location, release_file)
end

#extract_release ⇒ void

This method returns an undefined value.

Extracts release tarball



79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/bosh/director/jobs/update_release.rb', line 79

def extract_release
  release_tgz = File.join(@tmp_release_dir,
                          Api::ReleaseManager::RELEASE_TGZ)

  result = Bosh::Exec.sh("tar -C #{@tmp_release_dir} -xzf #{release_tgz} 2>&1", :on_error => :return)
  if result.failed?
    logger.error("Extracting release archive failed in dir #{@tmp_release_dir}, " +
                 "tar returned #{result.exit_status}, " +
                 "output: #{result.output}")
    raise ReleaseInvalidArchive, "Extracting release archive failed. Check task debug log for details."
  end
ensure
  if release_tgz && File.exists?(release_tgz)
    FileUtils.rm(release_tgz)
  end
end

#normalize_manifest ⇒ void

This method returns an undefined value.

Normalizes release manifest, so all names, versions, and checksums are Strings.



166
167
168
169
170
171
 
# File 'lib/bosh/director/jobs/update_release.rb', line 166

def normalize_manifest
  Bosh::Director.hash_string_vals(@manifest, 'name', 'version')

  @manifest['packages'].each { |p| Bosh::Director.hash_string_vals(p, 'name', 'version', 'sha1') }
  @manifest['jobs'].each { |j| Bosh::Director.hash_string_vals(j, 'name', 'version', 'sha1') }
end

#perform ⇒ void

This method returns an undefined value.

Extracts release tarball, verifies release manifest and saves release in DB



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/bosh/director/jobs/update_release.rb', line 45

def perform
  logger.info("Processing update release")
  if @rebase
    logger.info("Release rebase will be performed")
  end

  single_step_stage("Downloading remote release") { download_remote_release } if @remote_release
  single_step_stage("Extracting release") { extract_release }
  single_step_stage("Verifying manifest") { verify_manifest }

  with_release_lock(@name) { process_release }

  if @rebase && @packages_unchanged && @jobs_unchanged
    raise DirectorError,
          "Rebase is attempted without any job or package changes"
  end

  "Created release `#{@name}/#{@version}'"
rescue Exception => e
  remove_release_version_model
  raise e
ensure
  if @tmp_release_dir && File.exists?(@tmp_release_dir)
    FileUtils.rm_rf(@tmp_release_dir)
  end
end

#process_jobs ⇒ void

This method returns an undefined value.

Finds job template definitions in release manifest and sorts them into two buckets: new and existing job templates, then creates new job template records in the database and points release version to existing ones.



387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
 
# File 'lib/bosh/director/jobs/update_release.rb', line 387

def process_jobs
  logger.info("Checking for new jobs in release")

  new_jobs = []
  existing_jobs = []

  @manifest["jobs"].each do |job_meta|
    filter = {:sha1 => job_meta["sha1"]}
    if job_meta["fingerprint"]
      filter[:fingerprint] = job_meta["fingerprint"]
      filter = filter.sql_or
    end

    # Checking whether we might have the same bits somewhere
    jobs = Models::Template.where(filter).all

    if @rebase
      substitutes = jobs.select do |job|
        job.release_id == @release_model.id &&
        job.name == job_meta["name"]
      end

      substitute = pick_best(substitutes, job_meta["version"])

      if substitute
        job_meta["version"] = substitute.version
        job_meta["sha1"] = substitute.sha1
        existing_jobs << [substitute, job_meta]
      else
        new_jobs << job_meta
      end

      next
    end

    template = jobs.find do |job|
      job.release_id == @release_model.id &&
      job.name == job_meta["name"] &&
      job.version == job_meta["version"]
    end

    if template.nil?
      new_jobs << job_meta
    else
      existing_jobs << [template, job_meta]
    end
  end

  create_jobs(new_jobs)
  use_existing_jobs(existing_jobs)
end

#process_packages ⇒ void

This method returns an undefined value.

Finds all package definitions in the manifest and sorts them into two buckets: new and existing packages, then creates new packages and points current release version to the existing packages.



205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
 
# File 'lib/bosh/director/jobs/update_release.rb', line 205

def process_packages
  logger.info("Checking for new packages in release")

  new_packages = []
  existing_packages = []

  @manifest["packages"].each do |package_meta|
    filter = {:sha1 => package_meta["sha1"]}
    if package_meta["fingerprint"]
      filter[:fingerprint] = package_meta["fingerprint"]
      filter = filter.sql_or
    end

    # Checking whether we might have the same bits somewhere
    packages = Models::Package.where(filter).all

    if packages.empty?
      new_packages << package_meta
      next
    end

    # Rebase is an interesting use case: we don't really care about
    # preserving the original package/job versions, so if we have a
    # checksum/fingerprint match, we can just substitute the original
    # package/job version with an existing one.
    if @rebase
      substitutes = packages.select do |package|
        package.release_id == @release_model.id &&
        package.name == package_meta["name"] &&
        package.dependency_set == Set.new(package_meta["dependencies"])
      end

      substitute = pick_best(substitutes, package_meta["version"])

      if substitute
        package_meta["version"] = substitute.version
        package_meta["sha1"] = substitute.sha1
        existing_packages << [substitute, package_meta]
        next
      end
    end

    # We can reuse an existing package as long as it
    # belongs to the same release and has the same name and version.
    existing_package = packages.find do |package|
      package.release_id == @release_model.id &&
      package.name == package_meta["name"] &&
      package.version == package_meta["version"]
      # NOT checking dependencies here b/c dependency change would
      # bump the package version anyway.
    end

    if existing_package
      existing_packages << [existing_package, package_meta]
    else
      # We found a package with the same checksum but different
      # (release, name, version) tuple, so we need to make a copy
      # of the package blob and create a new db entry for it
      package = packages.first
      package_meta["blobstore_id"] = package.blobstore_id
      new_packages << package_meta
    end
  end

  create_packages(new_packages)
  use_existing_packages(existing_packages)
end

#process_release ⇒ void

This method returns an undefined value.

Processes uploaded release, creates jobs and packages in DB if needed



114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
 
# File 'lib/bosh/director/jobs/update_release.rb', line 114

def process_release
  @release_model = Models::Release.find_or_create(:name => @name)
  if @rebase
    @version = next_release_version
  end

  version_attrs = {
    :release => @release_model,
    :version => @version
  }
  version_attrs[:uncommitted_changes] = @uncommitted_changes if @uncommitted_changes
  version_attrs[:commit_hash] = @commit_hash if @commit_hash

  @release_version_model = Models::ReleaseVersion.new(version_attrs)
  unless @release_version_model.valid?
    raise ReleaseAlreadyExists,
          "Release `#{@name}/#{@version}' already exists"
  end

  @release_version_model.save

  single_step_stage("Resolving package dependencies") do
    resolve_package_dependencies(@manifest["packages"])
  end

  @packages = {}
  process_packages
  process_jobs

  unless @package_rebase_mapping.empty?
    event_log.begin_stage(
      "Rebased packages", @package_rebase_mapping.size)
    @package_rebase_mapping.each_pair do |name, transition|
      event_log.track("#{name}: #{transition}") {}
    end
  end

  unless @job_rebase_mapping.empty?
    event_log.begin_stage(
      "Rebased jobs", @job_rebase_mapping.size)
    @job_rebase_mapping.each_pair do |name, transition|
      event_log.track("#{name}: #{transition}") {}
    end
  end

  event_log.begin_stage("Release has been created", 1)
  event_log.track("#{@name}/#{@version}") {}
end

#register_package(package) ⇒ void

This method returns an undefined value.

Marks package model as used by release version model

Parameters:



377
378
379
380
 
# File 'lib/bosh/director/jobs/update_release.rb', line 377

def register_package(package)
  @packages[package.name] = package
  @release_version_model.add_package(package)
end

#register_template(template) ⇒ void

This method returns an undefined value.

Marks job template model as being used by release version

Parameters:



591
592
593
 
# File 'lib/bosh/director/jobs/update_release.rb', line 591

def register_template(template)
  @release_version_model.add_template(template)
end

#resolve_package_dependencies(packages) ⇒ void

This method returns an undefined value.

Resolves package dependencies, makes sure there are no cycles and all dependencies are present



176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
 
# File 'lib/bosh/director/jobs/update_release.rb', line 176

def resolve_package_dependencies(packages)
  packages_by_name = {}
  packages.each do |package|
    packages_by_name[package["name"]] = package
    package["dependencies"] ||= []
  end
  dependency_lookup = lambda do |package_name|
    packages_by_name[package_name]["dependencies"]
  end
  result = CycleHelper.check_for_cycle(packages_by_name.keys,
                                       :connected_vertices => true,
                                       &dependency_lookup)

  packages.each do |package|
    name = package["name"]
    dependencies = package["dependencies"]

    logger.info("Resolving package dependencies for `#{name}', " +
                "found: #{dependencies.pretty_inspect}")
    package["dependencies"] = result[:connected_vertices][name]
    logger.info("Resolved package dependencies for `#{name}', " +
                "to: #{dependencies.pretty_inspect}")
  end
end

#use_existing_jobs(jobs) ⇒ void

This method returns an undefined value.

Parameters:



559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
 
# File 'lib/bosh/director/jobs/update_release.rb', line 559

def use_existing_jobs(jobs)
  return if jobs.empty?

  n_jobs = jobs.size
  event_log.begin_stage("Processing #{n_jobs} existing " +
                        "job#{n_jobs > 1 ? "s" : ""}", 1)

  event_log.track("Verifying checksums") do
    jobs.each do |template, job_meta|
      job_desc = "#{template.name}/#{template.version}"

      logger.info("Job `#{job_desc}' already exists, " +
                  "verifying checksum")

      expected = template.sha1
      received = job_meta["sha1"]

      if expected != received
        raise ReleaseExistingJobHashMismatch,
              "`#{job_desc}' checksum mismatch, " +
              "expected #{expected} but received #{received}"
      end

      logger.info("Job `#{job_desc}' verified")
      register_template(template)
    end
  end
end

#use_existing_packages(packages) ⇒ Object

Points release DB model to existing packages described by given metadata

Parameters:

  • packages (Array<Array>) —

    Existing packages metadata



295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
 
# File 'lib/bosh/director/jobs/update_release.rb', line 295

def use_existing_packages(packages)
  return if packages.empty?

  n_packages = packages.size
  event_log.begin_stage("Processing #{n_packages} existing " +
                        "package#{n_packages > 1 ? "s" : ""}", 1)

  event_log.track("Verifying checksums") do
    packages.each do |package, package_meta|
      package_desc = "#{package.name}/#{package.version}"
      logger.info("Package `#{package_desc}' already exists, " +
                  "verifying checksum")

      expected = package.sha1
      received = package_meta["sha1"]

      if expected != received
        raise ReleaseExistingPackageHashMismatch,
              "`#{package_desc}' checksum mismatch, " +
                "expected #{expected} but received #{received}"
      end
      logger.info("Package `#{package_desc}' verified")
      register_package(package)
    end
  end
end

#verify_manifest ⇒ void

This method returns an undefined value.



97
98
99
100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/bosh/director/jobs/update_release.rb', line 97

def verify_manifest
  manifest_file = File.join(@tmp_release_dir, "release.MF")
  unless File.file?(manifest_file)
    raise ReleaseManifestNotFound, "Release manifest not found"
  end

  @manifest = Psych.load_file(manifest_file)
  normalize_manifest

  @name = @manifest["name"]
  @version = @manifest["version"]
  @commit_hash = @manifest.fetch("commit_hash", nil)
  @uncommitted_changes = @manifest.fetch("uncommitted_changes", nil)
end