Class: Chef::CookbookSynchronizer

Inherits:
Object
  • Object
show all
Defined in:
lib/chef/cookbook/synchronizer.rb

Overview

Synchronizes the locally cached copies of cookbooks with the files on the server.

Defined Under Namespace

Classes: CookbookFile

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(cookbooks_by_name, events) ⇒ CookbookSynchronizer

Returns a new instance of CookbookSynchronizer.



69
70
71
72
73
74
 
# File 'lib/chef/cookbook/synchronizer.rb', line 69

def initialize(cookbooks_by_name, events)
  @cookbooks_by_name, @events = cookbooks_by_name, events

  @cookbook_full_file_paths = {}
  @remove_obsoleted_files = true
end

Instance Attribute Details

#remove_obsoleted_files ⇒ Object

Returns the value of attribute remove_obsoleted_files.



67
68
69
 
# File 'lib/chef/cookbook/synchronizer.rb', line 67

def remove_obsoleted_files
  @remove_obsoleted_files
end

Instance Method Details

#cache ⇒ Object 



76
77
78
 
# File 'lib/chef/cookbook/synchronizer.rb', line 76

def cache
  Chef::FileCache
end

#cached_copy_up_to_date?(local_path, expected_checksum) ⇒ Boolean

Returns:

  • (Boolean) 


268
269
270
271
272
273
274
275
276
277
278
279
 
# File 'lib/chef/cookbook/synchronizer.rb', line 268

def cached_copy_up_to_date?(local_path, expected_checksum)
  if Chef::Config[:skip_cookbook_sync]
    Chef::Log.warn "skipping cookbook synchronization!  DO NOT LEAVE THIS ENABLED IN PRODUCTION!!!"
    return true
  end
  if cache.has_key?(local_path)
    current_checksum = CookbookVersion.checksum_cookbook_file(cache.load(local_path, false))
    expected_checksum == current_checksum
  else
    false
  end
end

#clear_obsoleted_cookbooks ⇒ Object

Iterates over cached cookbooks’ files, removing files belonging to cookbooks that don’t appear in cookbook_hash



214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/chef/cookbook/synchronizer.rb', line 214

def clear_obsoleted_cookbooks
  @events.cookbook_clean_start

  if remove_obsoleted_files
    remove_old_cookbooks
  else
    Chef::Log.info("Skipping removal of obsoleted cookbooks from the cache")
    CookbookCacheCleaner.instance.skip_removal = true
  end

  remove_deleted_files

  @events.cookbook_clean_complete
end

#cookbook_count ⇒ Object 



88
89
90
 
# File 'lib/chef/cookbook/synchronizer.rb', line 88

def cookbook_count
  @cookbooks_by_name.size
end

#cookbook_names ⇒ Object 



80
81
82
 
# File 'lib/chef/cookbook/synchronizer.rb', line 80

def cookbook_names
  @cookbooks_by_name.keys
end

#cookbook_segment(cookbook_name, segment) ⇒ Object 



96
97
98
 
# File 'lib/chef/cookbook/synchronizer.rb', line 96

def cookbook_segment(cookbook_name, segment)
  @cookbooks_by_name[cookbook_name].files_for(segment)
end

#cookbooks ⇒ Object 



84
85
86
 
# File 'lib/chef/cookbook/synchronizer.rb', line 84

def cookbooks
  @cookbooks_by_name.values
end

#download_file(url, destination) ⇒ Object

Unconditionally download the file from the given URL. File will be downloaded to the path destination which is relative to the Chef file cache root.



284
285
286
287
288
289
 
# File 'lib/chef/cookbook/synchronizer.rb', line 284

def download_file(url, destination)
  raw_file = server_api.streaming_request(url)

  Chef::Log.info("Storing updated #{destination} in the cache.")
  cache.move_to(raw_file.path, destination)
end

#ensure_cookbook_paths ⇒ Object 



235
236
237
238
239
240
 
# File 'lib/chef/cookbook/synchronizer.rb', line 235

def ensure_cookbook_paths
  cookbooks.each do |cookbook|
    cb_dir = File.join(Chef::Config[:file_cache_path], "cookbooks", cookbook.name)
    cookbook.root_paths = Array(cb_dir)
  end
end

#files ⇒ Object 



100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/chef/cookbook/synchronizer.rb', line 100

def files
  exclude = unless Chef::Config[:no_lazy_load]
              [ :files, :templates ]
            else
              []
            end

  @files ||= cookbooks.inject([]) do |memo, cookbook|
    cookbook.each_file(excluded_parts: exclude) do |manifest_record|
      memo << CookbookFile.new(cookbook, manifest_record)
    end
    memo
  end
end

#files_by_cookbook ⇒ Object 



115
116
117
 
# File 'lib/chef/cookbook/synchronizer.rb', line 115

def files_by_cookbook
  files.group_by { |file| file.cookbook }
end

#files_remaining_by_cookbook ⇒ Object 



119
120
121
122
123
124
125
126
 
# File 'lib/chef/cookbook/synchronizer.rb', line 119

def files_remaining_by_cookbook
  @files_remaining_by_cookbook ||= begin
    files_by_cookbook.inject({}) do |memo, (cookbook, files)|
      memo[cookbook] = files.size
      memo
    end
  end
end

#have_cookbook?(cookbook_name) ⇒ Boolean

Returns:

  • (Boolean) 


92
93
94
 
# File 'lib/chef/cookbook/synchronizer.rb', line 92

def have_cookbook?(cookbook_name)
  @cookbooks_by_name.key?(cookbook_name)
end

#mark_cached_file_valid(cache_filename) ⇒ Object

Marks the given file as valid (non-stale).



292
293
294
 
# File 'lib/chef/cookbook/synchronizer.rb', line 292

def mark_cached_file_valid(cache_filename)
  CookbookCacheCleaner.instance.mark_file_as_valid(cache_filename)
end

#mark_file_synced(file) ⇒ Object 



128
129
130
131
132
133
134
 
# File 'lib/chef/cookbook/synchronizer.rb', line 128

def mark_file_synced(file)
  files_remaining_by_cookbook[file.cookbook] -= 1

  if files_remaining_by_cookbook[file.cookbook] == 0
    @events.synchronized_cookbook(file.cookbook.name, file.cookbook)
  end
end

#remove_deleted_files ⇒ Object

remove deleted files in cookbooks that are being used on the node



196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/chef/cookbook/synchronizer.rb', line 196

def remove_deleted_files
  cache.find(File.join(%w{cookbooks ** {*,.*}})).each do |cache_file|
    md = cache_file.match(/^cookbooks\/([^\/]+)\/([^\/]+)\/(.*)/)
    next unless md
    ( cookbook_name, segment, file ) = md[1..3]
    if have_cookbook?(cookbook_name)
      manifest_segment = cookbook_segment(cookbook_name, segment)
      if manifest_segment.select { |manifest_record| manifest_record["path"] == "#{segment}/#{file}" }.empty?
        Chef::Log.info("Removing #{cache_file} from the cache; its is no longer in the cookbook manifest.")
        cache.delete(cache_file)
        @events.removed_cookbook_file(cache_file)
      end
    end
  end
end

#remove_old_cookbooks ⇒ Object

remove cookbooks that are not referenced in the expanded run_list at all (if we have an override run_list we may not want to do this)



184
185
186
187
188
189
190
191
192
193
 
# File 'lib/chef/cookbook/synchronizer.rb', line 184

def remove_old_cookbooks
  cache.find(File.join(%w{cookbooks ** {*,.*}})).each do |cache_file|
    cache_file =~ /^cookbooks\/([^\/]+)\//
    unless have_cookbook?($1)
      Chef::Log.info("Removing #{cache_file} from the cache; its cookbook is no longer needed on this client.")
      cache.delete(cache_file)
      @events.removed_cookbook_file(cache_file)
    end
  end
end

#save_full_file_path(file, full_path) ⇒ Object

Saves the full_path to the file of the cookbook to be updated in the manifest later



177
178
179
180
 
# File 'lib/chef/cookbook/synchronizer.rb', line 177

def save_full_file_path(file, full_path)
  @cookbook_full_file_paths[file.cookbook] ||= []
  @cookbook_full_file_paths[file.cookbook] << full_path
end

#server_api ⇒ Object 



296
297
298
 
# File 'lib/chef/cookbook/synchronizer.rb', line 296

def server_api
  Thread.current[:server_api] ||= Chef::ServerAPI.new(Chef::Config[:chef_server_url], keepalives: true)
end

#sync_cookbooks ⇒ Object

Synchronizes all the cookbooks from the chef-server. )

Returns

true

Always returns true



140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/chef/cookbook/synchronizer.rb', line 140

def sync_cookbooks
  Chef::Log.info("Loading cookbooks [#{cookbooks.map { |ckbk| ckbk.name + '@' + ckbk.version }.join(', ')}]")
  Chef::Log.debug("Cookbooks detail: #{cookbooks.inspect}")

  clear_obsoleted_cookbooks

  queue = Chef::Util::ThreadedJobQueue.new

  files.each do |file|
    queue << lambda do |lock|
      full_file_path = sync_file(file)

      lock.synchronize do
        # Save the full_path of the downloaded file to be restored in the manifest later
        save_full_file_path(file, full_file_path)
        mark_file_synced(file)
      end
    end
  end

  @events.cookbook_sync_start(cookbook_count)
  queue.process(Chef::Config[:cookbook_sync_threads])
  # Ensure that cookbooks know where they're rooted at, for manifest purposes.
  ensure_cookbook_paths
  # Update the full file paths in the manifest
  update_cookbook_filenames

rescue Exception => e
  @events.cookbook_sync_failed(cookbooks, e)
  raise
else
  @events.cookbook_sync_complete
  true
end

#sync_file(file) ⇒ Object

Sync an individual file if needed. If there is an up to date copy locally, nothing is done. Updates file‘s manifest with the full path to the cached file.

Arguments

file<CookbookFile>

Returns

Full path to the cached file as a String



250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
 
# File 'lib/chef/cookbook/synchronizer.rb', line 250

def sync_file(file)
  cache_filename = File.join("cookbooks", file.cookbook.name, file.manifest_record["path"])
  mark_cached_file_valid(cache_filename)

  # If the checksums are different between on-disk (current) and on-server
  # (remote, per manifest), do the update. This will also execute if there
  # is no current checksum.
  if !cached_copy_up_to_date?(cache_filename, file.manifest_record["checksum"])
    download_file(file.manifest_record["url"], cache_filename)
    @events.updated_cookbook_file(file.cookbook.name, cache_filename)
  else
    Chef::Log.debug("Not storing #{cache_filename}, as the cache is up to date.")
  end

  # Load the file in the cache and return the full file path to the loaded file
  cache.load(cache_filename, false)
end

#update_cookbook_filenames ⇒ Object 



229
230
231
232
233
 
# File 'lib/chef/cookbook/synchronizer.rb', line 229

def update_cookbook_filenames
  @cookbook_full_file_paths.each do |cookbook, full_paths|
    cookbook.all_files = full_paths
  end
end