Class: Puppet::Modulebuilder::Builder

Inherits:
Object
  • Object
show all
Defined in:
lib/puppet/modulebuilder/builder.rb

Overview

Class to build Puppet Modules from source

Constant Summary collapse

DEFAULT_IGNORED = 
[
  '/.*',
  '/pkg/',
  '~*',
  '/coverage',
  '/checksums.json',
  '/REVISION',
  '/spec/fixtures/modules/',
  '/vendor/'
].freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source, destination = nil, logger = nil) ⇒ Builder

Returns a new instance of Builder.



21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/puppet/modulebuilder/builder.rb', line 21

def initialize(source, destination = nil, logger = nil)
  unless logger.nil? || logger.is_a?(Logger)
    raise ArgumentError,
          format('logger is expected to be nil or a Logger. Got %<klass>s',
                 klass: logger.class)
  end

  @source_validated = false
  @source = source
  @destination = destination.nil? ? File.join(source, 'pkg') : destination
  @logger = logger.nil? ? ::Logger.new(File.open(File::NULL, 'w')) : logger
end

Instance Attribute Details

#destination ⇒ Object (readonly)

Returns the value of attribute destination.



19
20
21
 
# File 'lib/puppet/modulebuilder/builder.rb', line 19

def destination
  @destination
end

#logger ⇒ Object (readonly)

Returns the value of attribute logger.



19
20
21
 
# File 'lib/puppet/modulebuilder/builder.rb', line 19

def logger
  @logger
end

#release_name ⇒ String

The release name is used for the build directory and resulting package file.

The default combines the module name and version into a Forge-compatible dash separated string. Unless you have an unusual use case this isn’t set manually.

Returns:

  • (String) 


336
337
338
339
340
341
 
# File 'lib/puppet/modulebuilder/builder.rb', line 336

def release_name
  @release_name ||= [
    metadata['name'],
    metadata['version']
  ].join('-')
end

Instance Method Details

#build ⇒ String

Build a module package from a module directory.

Returns:

  • (String) —

    The path to the built package file.



46
47
48
49
50
51
52
53
54
55
 
# File 'lib/puppet/modulebuilder/builder.rb', line 46

def build
  create_build_dir

  stage_module_in_build_dir
  build_package

  package_file
ensure
  cleanup_build_dir
end

#build_context ⇒ Object 



65
66
67
68
69
70
 
# File 'lib/puppet/modulebuilder/builder.rb', line 65

def build_context
  {
    parent_dir: destination,
    build_dir_name: release_name
  }.freeze
end

#build_dir ⇒ Object

Return the path to the temporary build directory, which will be placed inside the target directory and match the release name

See Also:



61
62
63
 
# File 'lib/puppet/modulebuilder/builder.rb', line 61

def build_dir
  @build_dir ||= File.join(build_context[:parent_dir], build_context[:build_dir_name])
end

#build_package ⇒ Object

Creates a gzip compressed tarball of the build directory.

If the destination package already exists, it will be removed before creating the new tarball.

Returns:

  • nil.



211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
 
# File 'lib/puppet/modulebuilder/builder.rb', line 211

def build_package
  require 'zlib'
  require 'minitar'
  require 'find'

  FileUtils.rm_f(package_file)

  # The chdir necessary us due to Minitar entry not be able to separate the filename
  # within the TAR versus the source filename to pack
  Dir.chdir(build_context[:parent_dir]) do
    gz = Zlib::GzipWriter.new(File.open(package_file, 'wb'))
    begin
      tar = Minitar::Output.new(gz)
      Find.find(build_context[:build_dir_name]) do |entry|
        entry_meta = {
          name: entry
        }

        orig_mode = File.stat(entry).mode
        min_mode = Minitar.dir?(entry) ? 0o755 : 0o644

        entry_meta[:mode] = orig_mode | min_mode

        if entry_meta[:mode] != orig_mode
          logger.debug(format('Updated permissions of packaged \'%<entry>s\' to %<new_mode>s', entry: entry,
                                                                                               new_mode: (entry_meta[:mode] & 0o7777).to_s(8)))
        end

        Minitar.pack_file(entry_meta, tar)
      end
    ensure
      tar.close
    end
  end
end

#cleanup_build_dir ⇒ Object

Remove the temporary build directory and all its contents from disk.

Returns:

  • nil.



283
284
285
 
# File 'lib/puppet/modulebuilder/builder.rb', line 283

def cleanup_build_dir
  FileUtils.rm_rf(build_dir, secure: true)
end

#copy_mtime(path) ⇒ Object 



132
133
134
135
136
137
138
139
140
141
 
# File 'lib/puppet/modulebuilder/builder.rb', line 132

def copy_mtime(path)
  require 'pathname'

  relative_path = Pathname.new(path).relative_path_from(Pathname.new(source))
  dest_path = File.join(build_dir, relative_path)

  validate_path_encoding!(relative_path.to_path)

  fileutils_touch(dest_path, mtime: file_stat(path).mtime)
end

#create_build_dir ⇒ Object

Create a temporary build directory where the files to be included in the package will be staged before building the tarball.

If the directory already exists, remove it first.



274
275
276
277
278
 
# File 'lib/puppet/modulebuilder/builder.rb', line 274

def create_build_dir
  cleanup_build_dir

  fileutils_mkdir_p(build_dir)
end

#ignore_file ⇒ String

Select the most appropriate ignore file in the module directory.

In order of preference, we first try ‘.pdkignore`, then `.pmtignore` and finally `.gitignore`.

Returns:

  • (String) —

    The path to the file containing the patterns of file paths to ignore.



178
179
180
181
182
183
184
 
# File 'lib/puppet/modulebuilder/builder.rb', line 178

def ignore_file
  @ignore_file ||= [
    File.join(source, '.pdkignore'),
    File.join(source, '.pmtignore'),
    File.join(source, '.gitignore')
  ].find { |file| file_exists?(file) && file_readable?(file) }
end

#ignored_files ⇒ PathSpec

Instantiate a new PathSpec class and populate it with the pattern(s) of files to be ignored.

Returns:

  • (PathSpec) —

    The populated ignore path matcher.



251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
 
# File 'lib/puppet/modulebuilder/builder.rb', line 251

def ignored_files
  require 'pathspec'

  @ignored_files ||=
    begin
      ignored = if ignore_file.nil?
                  PathSpec.new
                else
                  PathSpec.new(read_file(ignore_file, open_args: 'rb:UTF-8'))
                end

      ignored = ignored.add("/#{File.basename(destination)}/") if File.realdirpath(destination).start_with?(File.realdirpath(source))

      DEFAULT_IGNORED.each { |r| ignored.add(r) }

      ignored
    end
end

#ignored_path?(path) ⇒ Boolean

Check if the given path matches one of the patterns listed in the ignore file.

Parameters:

  • path (String) —

    The path to be checked.

Returns:

  • (Boolean) —

    true if the path matches and should be ignored.



149
150
151
152
153
 
# File 'lib/puppet/modulebuilder/builder.rb', line 149

def ignored_path?(path)
  path = "#{path}/" if File.directory?(path)

  ignored_files.match_path(path, source)
end

#metadata ⇒ Hash{String => Object}

Read and parse the values from metadata.json for the module that is being built.

Returns:

  • (Hash{String => Object}) —

    The hash of metadata values.



291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
 
# File 'lib/puppet/modulebuilder/builder.rb', line 291

def metadata
  return @metadata unless @metadata.nil?

  metadata_json_path = File.join(source, 'metadata.json')

  unless file_exists?(metadata_json_path)
    raise ArgumentError,
          format("'%<file>s' does not exist or is not a file.",
                 file: metadata_json_path)
  end

  unless file_readable?(metadata_json_path)
    raise ArgumentError,
          format("Unable to open '%<file>s' for reading.",
                 file: metadata_json_path)
  end

  require 'json'
  begin
    @metadata = JSON.parse(read_file(metadata_json_path))
  rescue JSON::JSONError => e
    raise ArgumentError, format('Invalid JSON in metadata.json: %<msg>s', msg: e.message)
  end
  @metadata.freeze
end

#package_already_exists? ⇒ Boolean

Verify if there is an existing package in the target directory and prompts the user if they want to overwrite it.

Returns:

  • (Boolean) 


324
325
326
 
# File 'lib/puppet/modulebuilder/builder.rb', line 324

def package_already_exists?
  file_exists?(package_file)
end

#package_file ⇒ Object

Return the path where the built package file will be written to.



318
319
320
 
# File 'lib/puppet/modulebuilder/builder.rb', line 318

def package_file
  @package_file ||= File.join(destination, "#{release_name}.tar.gz")
end

#source ⇒ String

The source to build the module from

Returns:

  • (String) 


36
37
38
39
40
41
 
# File 'lib/puppet/modulebuilder/builder.rb', line 36

def source
  return @source if @source_validated

  validate_source!
  @source = File.realpath(@source)
end

#stage_module_in_build_dir ⇒ Object

Iterate through all the files and directories in the module and stage them into the temporary build directory (unless ignored).

Returns:

  • nil



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/puppet/modulebuilder/builder.rb', line 76

def stage_module_in_build_dir
  require 'find'

  directories = [source]

  staged = Find.find(source) do |path|
    next if path == source

    if ignored_path?(path)
      logger.debug("Ignoring #{path} from the build")
      Find.prune
    else
      logger.debug("Staging #{path} for the build")
      directories << path if file_directory?(path)
      stage_path(path)
    end
  end

  # Reset directory mtimes. This must happen after the files have been
  # copied since that modifies a directory's mtime
  directories.each do |directory|
    copy_mtime(directory)
  end

  staged
end

#stage_path(path) ⇒ Object

Stage a file or directory from the module into the build directory.

Parameters:

  • path (String) —

    The path to the file or directory.

Returns:

  • nil.



108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
 
# File 'lib/puppet/modulebuilder/builder.rb', line 108

def stage_path(path)
  require 'pathname'

  relative_path = Pathname.new(path).relative_path_from(Pathname.new(source))
  dest_path = File.join(build_dir, relative_path)

  validate_path_encoding!(relative_path.to_path)

  begin
    if file_directory?(path)
      fileutils_mkdir_p(dest_path, mode: file_stat(path).mode)
    elsif file_symlink?(path)
      warn_symlink(path)
    else
      validate_ustar_path!(relative_path.to_path)
      fileutils_cp(path, dest_path, preserve: true)
    end
  rescue ArgumentError => e
    raise format(
      '%<message>s Rename the file or exclude it from the package by adding it to the .pdkignore file in your module.', message: e.message
    )
  end
end

#validate_path_encoding!(path) ⇒ nil

Checks if the path contains any non-ASCII characters.

Java will throw an error when it encounters a path containing characters that are not supported by the hosts locale. In order to maximise compatibility we limit the paths to contain only ASCII characters, which should be part of any locale character set.

Parameters:

  • path (String) —

    the relative path to be added to the tar file.

Returns:

  • (nil)

Raises:

  • (ArgumentError) —

    if the path contains non-ASCII characters.



198
199
200
201
202
203
 
# File 'lib/puppet/modulebuilder/builder.rb', line 198

def validate_path_encoding!(path)
  return unless /[^\x00-\x7F]/.match?(path)

  raise ArgumentError, format("'%<path>s' can only include ASCII characters in its path or " \
                              'filename in order to be compatible with a wide range of hosts.', path: path)
end

#validate_ustar_path!(path) ⇒ nil

Checks if the path length will fit into the POSIX.1-1998 (ustar) tar header format.

POSIX.1-2001 (which allows paths of infinite length) was adopted by GNU tar in 2004 and is supported by minitar 0.7 and above. Unfortunately much of the Puppet ecosystem still uses minitar 0.6.1.

POSIX.1-1998 tar format does not allow for paths greater than 256 bytes, or paths that can’t be split into a prefix of 155 bytes (max) and a suffix of 100 bytes (max).

This logic was pretty much copied from the private method Archive::Tar::Minitar::Writer#split_name.

Parameters:

  • path (String) —

    the relative path to be added to the tar file.

Returns:

  • (nil)

Raises:

  • (ArgumentError) —

    if the path is too long or could not be split.



364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
 
# File 'lib/puppet/modulebuilder/builder.rb', line 364

def validate_ustar_path!(path)
  raise ArgumentError, format("The path '%<path>s' is longer than 256 bytes.", path: path) if path.bytesize > 256

  if path.bytesize <= 100
    prefix = ''
  else
    parts = path.split(File::SEPARATOR)
    newpath = parts.pop
    nxt = ''

    loop do
      nxt = parts.pop || ''
      break if newpath.bytesize + 1 + nxt.bytesize >= 100

      newpath = File.join(nxt, newpath)
    end

    prefix = File.join(*parts, nxt)
    path = newpath
  end

  return unless path.bytesize > 100 || prefix.bytesize > 155

  raise ArgumentError, \
        format("'%<path>s' could not be split at a directory separator into two " \
               'parts, the first having a maximum length of 155 bytes and the ' \
               'second having a maximum length of 100 bytes.', path: path)
end

#warn_symlink(path) ⇒ Object

Warn the user about a symlink that would have been included in the built package.

Parameters:

  • path (String) —

    The relative or absolute path to the symlink.

Returns:

  • nil.



161
162
163
164
165
166
167
168
169
 
# File 'lib/puppet/modulebuilder/builder.rb', line 161

def warn_symlink(path)
  require 'pathname'

  symlink_path = Pathname.new(path)
  module_path = Pathname.new(source)

  logger.warn format('Symlinks in modules are not supported and will not be included in the package. Please investigate symlink %<from>s -> %<to>s.',
                     from: symlink_path.relative_path_from(module_path), to: symlink_path.realpath.relative_path_from(module_path))
end