Class: Chef::CookbookSynchronizer

Inherits:
Object
  • Object
show all
Defined in:
lib/chef/cookbook/synchronizer.rb

Overview

Synchronizes the locally cached copies of cookbooks with the files on the server.

Defined Under Namespace

Classes: CookbookFile

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(cookbooks_by_name, events) ⇒ CookbookSynchronizer

Returns a new instance of CookbookSynchronizer.


86
87
88
89
90
91
92
93
# File 'lib/chef/cookbook/synchronizer.rb', line 86

def initialize(cookbooks_by_name, events)
  @cookbooks_by_name, @events = cookbooks_by_name, events

  @cookbook_full_file_paths = {}
  @remove_obsoleted_files = true

  @lazy_files = {}
end

Instance Attribute Details

#remove_obsoleted_filesObject

Returns the value of attribute remove_obsoleted_files


84
85
86
# File 'lib/chef/cookbook/synchronizer.rb', line 84

def remove_obsoleted_files
  @remove_obsoleted_files
end

Instance Method Details

#cacheObject


95
96
97
# File 'lib/chef/cookbook/synchronizer.rb', line 95

def cache
  Chef::FileCache
end

#cached_copy_up_to_date?(local_path, expected_checksum) ⇒ Boolean

Returns:

  • (Boolean)

298
299
300
301
302
303
304
305
306
307
# File 'lib/chef/cookbook/synchronizer.rb', line 298

def cached_copy_up_to_date?(local_path, expected_checksum)
  return true if Chef::Config[:skip_cookbook_sync]

  if cache.key?(local_path)
    current_checksum = CookbookVersion.checksum_cookbook_file(cache.load(local_path, false))
    expected_checksum == current_checksum
  else
    false
  end
end

#clear_obsoleted_cookbooksObject

Iterates over cached cookbooks' files, removing files belonging to cookbooks that don't appear in +cookbook_hash+


240
241
242
243
244
245
246
247
248
249
250
251
252
253
# File 'lib/chef/cookbook/synchronizer.rb', line 240

def clear_obsoleted_cookbooks
  @events.cookbook_clean_start

  if remove_obsoleted_files
    remove_old_cookbooks
  else
    Chef::Log.info("Skipping removal of obsoleted cookbooks from the cache")
    CookbookCacheCleaner.instance.skip_removal = true
  end

  remove_deleted_files

  @events.cookbook_clean_complete
end

#cookbook_countObject


107
108
109
# File 'lib/chef/cookbook/synchronizer.rb', line 107

def cookbook_count
  @cookbooks_by_name.size
end

#cookbook_namesObject


99
100
101
# File 'lib/chef/cookbook/synchronizer.rb', line 99

def cookbook_names
  @cookbooks_by_name.keys
end

#cookbook_segment(cookbook_name, segment) ⇒ Object


115
116
117
# File 'lib/chef/cookbook/synchronizer.rb', line 115

def cookbook_segment(cookbook_name, segment)
  @cookbooks_by_name[cookbook_name].files_for(segment)
end

#cookbooksObject


103
104
105
# File 'lib/chef/cookbook/synchronizer.rb', line 103

def cookbooks
  @cookbooks_by_name.values
end

#download_file(url, destination) ⇒ Object

Unconditionally download the file from the given URL. File will be downloaded to the path +destination+ which is relative to the Chef file cache root.


312
313
314
315
316
317
# File 'lib/chef/cookbook/synchronizer.rb', line 312

def download_file(url, destination)
  raw_file = server_api.streaming_request(url)

  Chef::Log.info("Storing updated #{destination} in the cache.")
  cache.move_to(raw_file.path, destination)
end

#ensure_cookbook_pathsObject


265
266
267
268
269
270
# File 'lib/chef/cookbook/synchronizer.rb', line 265

def ensure_cookbook_paths
  cookbooks.each do |cookbook|
    cb_dir = File.join(Chef::Config[:file_cache_path], "cookbooks", cookbook.name)
    cookbook.root_paths = Array(cb_dir)
  end
end

#filesObject


119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
# File 'lib/chef/cookbook/synchronizer.rb', line 119

def files
  lazy = unless Chef::Config[:no_lazy_load]
           %w{ files templates }
         else
           []
         end

  @files ||= cookbooks.inject([]) do |memo, cookbook|
    cookbook.each_file do |manifest_record|
      part = manifest_record[:name].split("/")[0]
      if lazy.include?(part)
        manifest_record[:lazy] = true
        @lazy_files[cookbook] ||= []
        @lazy_files[cookbook] << manifest_record
      else
        memo << CookbookFile.new(cookbook, manifest_record)
      end
    end
    memo
  end
end

#files_by_cookbookObject


141
142
143
# File 'lib/chef/cookbook/synchronizer.rb', line 141

def files_by_cookbook
  files.group_by(&:cookbook)
end

#files_remaining_by_cookbookObject


145
146
147
148
149
150
# File 'lib/chef/cookbook/synchronizer.rb', line 145

def files_remaining_by_cookbook
  @files_remaining_by_cookbook ||= files_by_cookbook.inject({}) do |memo, (cookbook, files)|
    memo[cookbook] = files.size
    memo
  end
end

#have_cookbook?(cookbook_name) ⇒ Boolean

Returns:

  • (Boolean)

111
112
113
# File 'lib/chef/cookbook/synchronizer.rb', line 111

def have_cookbook?(cookbook_name)
  @cookbooks_by_name.key?(cookbook_name)
end

#mark_cached_file_valid(cache_filename) ⇒ Object

Marks the given file as valid (non-stale).


320
321
322
# File 'lib/chef/cookbook/synchronizer.rb', line 320

def mark_cached_file_valid(cache_filename)
  CookbookCacheCleaner.instance.mark_file_as_valid(cache_filename)
end

#mark_file_synced(file) ⇒ Object


152
153
154
155
156
157
158
# File 'lib/chef/cookbook/synchronizer.rb', line 152

def mark_file_synced(file)
  files_remaining_by_cookbook[file.cookbook] -= 1

  if files_remaining_by_cookbook[file.cookbook] == 0
    @events.synchronized_cookbook(file.cookbook.name, file.cookbook)
  end
end

#remove_deleted_filesObject

remove deleted files in cookbooks that are being used on the node


221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
# File 'lib/chef/cookbook/synchronizer.rb', line 221

def remove_deleted_files
  cache.find(File.join(%w{cookbooks ** {*,.*}})).each do |cache_file|
    md = cache_file.match(%r{^cookbooks/([^/]+)/([^/]+)/(.*)})
    next unless md

    ( cookbook_name, segment, file ) = md[1..3]
    if have_cookbook?(cookbook_name)
      manifest_segment = cookbook_segment(cookbook_name, segment)
      if manifest_segment.select { |manifest_record| manifest_record["path"] == "#{segment}/#{file}" }.empty?
        Chef::Log.info("Removing #{cache_file} from the cache; its is no longer in the cookbook manifest.")
        cache.delete(cache_file)
        @events.removed_cookbook_file(cache_file)
      end
    end
  end
end

#remove_old_cookbooksObject

remove cookbooks that are not referenced in the expanded run_list at all (if we have an override run_list we may not want to do this)


209
210
211
212
213
214
215
216
217
218
# File 'lib/chef/cookbook/synchronizer.rb', line 209

def remove_old_cookbooks
  cache.find(File.join(%w{cookbooks ** {*,.*}})).each do |cache_file|
    cache_file =~ %r{^cookbooks/([^/]+)/}
    unless have_cookbook?($1)
      Chef::Log.info("Removing #{cache_file} from the cache; its cookbook is no longer needed on this client.")
      cache.delete(cache_file)
      @events.removed_cookbook_file(cache_file)
    end
  end
end

#save_full_file_path(file, full_path) ⇒ Object

Saves the full_path to the file of the cookbook to be updated in the manifest later


202
203
204
205
# File 'lib/chef/cookbook/synchronizer.rb', line 202

def save_full_file_path(file, full_path)
  @cookbook_full_file_paths[file.cookbook] ||= []
  @cookbook_full_file_paths[file.cookbook] << full_path
end

#server_apiObject


324
325
326
# File 'lib/chef/cookbook/synchronizer.rb', line 324

def server_api
  Thread.current[:server_api] ||= Chef::ServerAPI.new(Chef::Config[:chef_server_url], keepalives: true)
end

#sync_cookbooksObject

Synchronizes all the cookbooks from the chef-server. ) === Returns true:: Always returns true


164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
# File 'lib/chef/cookbook/synchronizer.rb', line 164

def sync_cookbooks
  Chef::Log.info("Loading cookbooks [#{cookbooks.map { |ckbk| ckbk.name + "@" + ckbk.version }.join(", ")}]")
  Chef::Log.trace("Cookbooks detail: #{cookbooks.inspect}")

  clear_obsoleted_cookbooks

  queue = Chef::Util::ThreadedJobQueue.new

  Chef::Log.warn("skipping cookbook synchronization! DO NOT LEAVE THIS ENABLED IN PRODUCTION!!!") if Chef::Config[:skip_cookbook_sync]
  files.each do |file|
    queue << lambda do |lock|
      full_file_path = sync_file(file)

      lock.synchronize do
        # Save the full_path of the downloaded file to be restored in the manifest later
        save_full_file_path(file, full_file_path)
        mark_file_synced(file)
      end
    end
  end

  @events.cookbook_sync_start(cookbook_count)
  queue.process(Chef::Config[:cookbook_sync_threads])
  # Ensure that cookbooks know where they're rooted at, for manifest purposes.
  ensure_cookbook_paths
  # Update the full file paths in the manifest
  update_cookbook_filenames

rescue Exception => e
  @events.cookbook_sync_failed(cookbooks, e)
  raise
else
  @events.cookbook_sync_complete
  true
end

#sync_file(file) ⇒ Object

Sync an individual file if needed. If there is an up to date copy locally, nothing is done. Updates +file+'s manifest with the full path to the cached file.

=== Arguments file === Returns Full path to the cached file as a String


280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
# File 'lib/chef/cookbook/synchronizer.rb', line 280

def sync_file(file)
  cache_filename = File.join("cookbooks", file.cookbook.name, file.manifest_record["path"])
  mark_cached_file_valid(cache_filename)

  # If the checksums are different between on-disk (current) and on-server
  # (remote, per manifest), do the update. This will also execute if there
  # is no current checksum.
  if !cached_copy_up_to_date?(cache_filename, file.manifest_record["checksum"])
    download_file(file.manifest_record["url"], cache_filename)
    @events.updated_cookbook_file(file.cookbook.name, cache_filename)
  else
    Chef::Log.trace("Not storing #{cache_filename}, as the cache is up to date.")
  end

  # Load the file in the cache and return the full file path to the loaded file
  cache.load(cache_filename, false)
end

#update_cookbook_filenamesObject


255
256
257
258
259
260
261
262
263
# File 'lib/chef/cookbook/synchronizer.rb', line 255

def update_cookbook_filenames
  @cookbook_full_file_paths.each do |cookbook, full_paths|
    cookbook.all_files = full_paths
  end

  @lazy_files.each do |cookbook, lazy_files|
    cookbook.cookbook_manifest.add_files_to_manifest(lazy_files)
  end
end