Class: Autobuild::ArchiveImporter

Inherits:
Importer
  • Object
show all
Defined in:
lib/autobuild/import/archive.rb

Constant Summary collapse

Plain =

rubocop:disable Naming/ConstantName The tarball is not compressed

0
Gzip =

The tarball is compressed with gzip

1
Bzip =

The tarball is compressed using bzip

2
Zip =

Not a tarball but a zip

3
TAR_OPTION =

rubocop:enable Naming/ConstantName

{
    Plain => '',
    Gzip => 'z',
    Bzip => 'j'
}.freeze
VALID_URI_SCHEMES =

Known URI schemes for url

%w[file http https ftp].freeze
WINDOWS_VALID_URI_SCHEMES =

Known URI schemes for url on windows

%w[file http https].freeze

Class Attribute Summary collapse

Instance Attribute Summary collapse

Attributes inherited from Importer

#interactive, #options, #post_hooks, #repository_id, #source_id

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from Importer

#add_post_hook, add_post_hook, #apply, cache_dirs, #call_patch, #currently_applied_patches, default_cache_dirs, default_cache_dirs=, #each_post_hook, each_post_hook, #execute_post_hooks, #fallback, fallback, #fingerprint, #import, #interactive?, #parse_patch_list, #patch, #patchdir, #patches, #patches_fingerprint, #patchlist, #perform_checkout, #perform_update, #retry_count, #retry_count=, #save_patch_state, set_cache_dirs, #supports_relocation?, #unapply, unset_cache_dirs, #update_retry_count

Constructor Details

#initialize(url, options = Hash.new) ⇒ ArchiveImporter

Creates a new importer which downloads url in cachedir and unpacks it. The following options are allowed:

:cachedir

the cache directory. Defaults to “#Autobuild.prefix/cache”

:archive_dir

the directory contained in the archive file. If set,

the importer will rename that directory to make it match
Package#srcdir
:no_subdirectory

the archive does not have the custom archive

subdirectory.
:retries

The number of retries for downloading

:timeout

The timeout (in seconds) used during downloading, it

defaults to 10s
:filename

Rename the archive to this filename (in cache) – will be

also used to infer the mode
:mode

The unpack mode: one of Zip, Bzip, Gzip or Plain, this is

usually automatically inferred from the filename



380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
 
# File 'lib/autobuild/import/archive.rb', line 380

def initialize(url, options = Hash.new)
    sourceopts, options = Kernel.filter_options(
        options,
        :source_id, :repository_id, :filename, :mode, :update_cached_file,
        :user, :password, :expected_digest
    )
    super(options)

    @filename = nil
    @update_cached_file = false
    @cachedir = @options[:cachedir] || ArchiveImporter.cachedir
    @retries  = @options[:retries] || ArchiveImporter.retries
    @timeout  = @options[:timeout] || ArchiveImporter.timeout
    relocate(url, sourceopts)
end

Class Attribute Details

.cachedir ⇒ Object

The directory in which downloaded files are saved

It defaults, if set, to the value returned by Importer.cache_dirs and falls back #prefix/cache



31
32
33
34
35
36
37
38
 
# File 'lib/autobuild/import/archive.rb', line 31

def cachedir
    if @cachedir then @cachedir
    elsif (cache_dirs = Importer.cache_dirs('archives'))
        @cachedir = cache_dirs.first
    else
        "#{Autobuild.prefix}/cache"
    end
end

.retries ⇒ Object

The number of time we should retry downloading if the underlying tool supports it (wget does).

It defaults to 1 as autobuild has its own retry mechanism



55
56
57
 
# File 'lib/autobuild/import/archive.rb', line 55

def retries
  @retries
end

.timeout ⇒ Object

The timeout (in seconds) used during downloading.

With wget, it is the timeout used for DNS resolution, connection and idle time (time without receiving data)

It defaults to 10s



49
50
51
 
# File 'lib/autobuild/import/archive.rb', line 49

def timeout
  @timeout
end

Instance Attribute Details

#cachedir ⇒ Object

The directory in which remote files are cached

Defaults to ArchiveImporter.cachedir



312
313
314
 
# File 'lib/autobuild/import/archive.rb', line 312

def cachedir
  @cachedir
end

#cachefile ⇒ Object (readonly)

The local file (either a downloaded file if url is not local, or url itself)



301
302
303
 
# File 'lib/autobuild/import/archive.rb', line 301

def cachefile
  @cachefile
end

#cachefile_digest ⇒ String (readonly)

The SHA1 digest of the current cachefile. It is updated only once the cachefile has been downloaded

Returns:

  • (String) —

    hexadecimal SHA1 digest of the file



306
307
308
 
# File 'lib/autobuild/import/archive.rb', line 306

def cachefile_digest
  @cachefile_digest
end

#filename ⇒ Object (readonly)

The filename that should be used locally (for remote files)

This is usually inferred by using the URL’s basename, but some download URLs do not allow this (for instance bitbucket tarballs)

Change it by calling #relocate



345
346
347
 
# File 'lib/autobuild/import/archive.rb', line 345

def filename
  @filename
end

#mode ⇒ Object (readonly)

The unpack mode. One of Zip, Bzip, Gzip or Plain



308
309
310
 
# File 'lib/autobuild/import/archive.rb', line 308

def mode
  @mode
end

#retries ⇒ Object

The number of time we should retry downloading if the underlying tool supports it (wget does).

It defaults to the global ArchiveImporter.retries



335
336
337
 
# File 'lib/autobuild/import/archive.rb', line 335

def retries
  @retries
end

#timeout ⇒ Object

The timeout (in seconds) used during downloading.

With wget, it is the timeout used for DNS resolution, connection and idle time (time without receiving data)

It defaults to the global ArchiveImporter.timeout



353
354
355
 
# File 'lib/autobuild/import/archive.rb', line 353

def timeout
  @timeout
end

#update_cached_file=(value) ⇒ Object (writeonly)

Sets the attribute update_cached_file

Parameters:

  • value —

    the value to set the attribute update_cached_file to.



101
102
103
 
# File 'lib/autobuild/import/archive.rb', line 101

def update_cached_file=(value)
  @update_cached_file = value
end

#url ⇒ Object (readonly)

The source URL



299
300
301
 
# File 'lib/autobuild/import/archive.rb', line 299

def url
  @url
end

Class Method Details

.auto_update=(flag) ⇒ Object 



96
97
98
 
# File 'lib/autobuild/import/archive.rb', line 96

def self.auto_update=(flag)
    @auto_update = flag
end

.auto_update? ⇒ Boolean

Tells the importer that the checkout should be automatically deleted on update, without asking the user

Returns:

  • (Boolean) —

    true if the archive importer should automatically delete the current checkout when the archive changed, false otherwise. The default is to set it to true if the AUTOBUILD_ARCHIVE_AUTOUPDATE environment variable is set to 1, and to false in all other cases



92
93
94
 
# File 'lib/autobuild/import/archive.rb', line 92

def self.auto_update?
    @auto_update
end

.filename_to_mode(filename) ⇒ Object

Returns the unpack mode from the file name



76
77
78
79
80
81
82
83
 
# File 'lib/autobuild/import/archive.rb', line 76

def self.filename_to_mode(filename)
    if (mode = find_mode_from_filename(filename))
        mode
    else
        raise "cannot infer the archive type from '#{filename}', "\
            "provide it explicitely with the mode: option"
    end
end

.find_mode_from_filename(filename) ⇒ Integer?

Returns the unpack mode from the file name

Returns:

  • (Integer, nil) —

    either one of the pack constants (Zip, Plain, …) or nil if it cannot be inferred

See Also:



66
67
68
69
70
71
72
73
 
# File 'lib/autobuild/import/archive.rb', line 66

def self.find_mode_from_filename(filename)
    case filename
    when /\.zip$/ then Zip
    when /\.tar$/ then Plain
    when /\.tar\.gz$|\.tgz$/ then Gzip
    when /\.bz2$/ then Bzip
    end
end

Instance Method Details

#archive_changed?(package) ⇒ Boolean

Returns true if the archive that has been used to checkout this package is different from the one we are supposed to checkout now

Returns:

  • (Boolean) 


475
476
477
478
 
# File 'lib/autobuild/import/archive.rb', line 475

def archive_changed?(package)
    checkout_digest = File.read(checkout_digest_stamp(package)).strip
    checkout_digest != cachefile_digest
end

#archive_dir ⇒ Object

The directory contained in the archive. If not set, we assume that it is the same than the source dir



327
328
329
 
# File 'lib/autobuild/import/archive.rb', line 327

def archive_dir
    @options[:archive_dir] || tardir
end

#checkout(package, options = Hash.new) ⇒ Object

:nodoc:



480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
 
# File 'lib/autobuild/import/archive.rb', line 480

def checkout(package, options = Hash.new) # :nodoc:
    options = Kernel.validate_options options,
                                      allow_interactive: true

    update_cache(package)

    # Check whether the archive file changed, and if that is the case
    # then ask the user about deleting the folder
    if File.file?(checkout_digest_stamp(package)) && archive_changed?(package)
        if ArchiveImporter.auto_update?
            response = 'yes'
        elsif options[:allow_interactive]
            package.progress_done
            package.message "The archive #{@url} is different from "\
                "the one currently checked out at #{package.srcdir}", :bold
            package.message "I will have to delete the current folder to go on "\
                "with the update"
            response = TTY::Prompt.new.ask "  Continue (yes or no) ? "\
                "If no, this update will be ignored, "\
                "which can lead to build problems.", convert: :bool
        else
            raise Autobuild::InteractionRequired, "importing #{package.name} "\
                "would have required user interaction and "\
                "allow_interactive is false"
        end

        if !response
            package.message "not updating #{package.srcdir}"
            package.progress_done
            return false
        else
            package.message "deleting #{package.srcdir} to update to new archive"
            FileUtils.rm_rf package.srcdir
            package.progress "checking out %s"
        end
    end

    # Un-apply any existing patch so that, when the files get
    # overwritten by the new checkout, the patch are re-applied
    patch(package, [])

    base_dir = File.dirname(package.srcdir)

    if mode == Zip
        main_dir = if @options[:no_subdirectory] then package.srcdir
                   else
                       base_dir
                   end

        FileUtils.mkdir_p base_dir
        cmd = ['-o', cachefile, '-d', main_dir]
        package.run(:import, Autobuild.tool('unzip'), *cmd)

        archive_dir = (self.archive_dir || File.basename(package.name))
        if archive_dir != File.basename(package.srcdir)
            FileUtils.rm_rf File.join(package.srcdir)
            FileUtils.mv File.join(base_dir, archive_dir), package.srcdir
        elsif !File.directory?(package.srcdir)
            raise Autobuild::Exception, "#{cachefile} does not contain "\
                "directory called #{File.basename(package.srcdir)}. "\
                "Did you forget to use the archive_dir option ?"
        end
    else
        FileUtils.mkdir_p package.srcdir
        cmd = ["x#{TAR_OPTION[mode]}f", cachefile, '-C', package.srcdir]
        cmd << '--strip-components=1' unless @options[:no_subdirectory]

        if Autobuild.windows?
            io = if mode == Plain
                     File.open(cachefile, 'r')
                 else
                     Zlib::GzipReader.open(cachefile)
                 end
            extract_tar_gz(io, package.srcdir)
        else
            package.run(:import, Autobuild.tool('tar'), *cmd)
        end
    end
    write_checkout_digest_stamp(package)
    true
rescue SubcommandFailed
    FileUtils.rm_f(cachefile) if cachefile != url.path
    raise
end

#checkout_digest_stamp(package) ⇒ Object 



463
464
465
 
# File 'lib/autobuild/import/archive.rb', line 463

def checkout_digest_stamp(package)
    File.join(package.srcdir, "archive-autobuild-stamp")
end

#download_from_url(package) ⇒ Object 



219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
 
# File 'lib/autobuild/import/archive.rb', line 219

def download_from_url(package)
    FileUtils.mkdir_p(cachedir)
    begin
        if %w[http https].include?(@url.scheme)
            if File.file?(cachefile)
                return false unless update_cached_file?

                cached_mtime = File.lstat(cachefile).mtime
            end

            updated = download_http(package, @url, "#{cachefile}.partial",
                                    user: @user, password: @password,
                                    current_time: cached_mtime)
            return false unless updated
        elsif Autobuild.bsd?
            return false unless update_needed?(package)

            package.run(:import, Autobuild.tool('curl'),
                        '-Lso', "#{cachefile}.partial", @url)
        else
            return false unless update_needed?(package)

            additional_options = []
            if (timeout = self.timeout)
                additional_options << "--timeout" << timeout
            end
            if (retries = self.retries)
                additional_options << "--tries" << retries
            end
            package.run(:import, Autobuild.tool('wget'), '-q', '-P', cachedir,
                        *additional_options, @url, '-O', "#{cachefile}.partial",
                        retry: true)
        end
    rescue Exception
        FileUtils.rm_f "#{cachefile}.partial"
        raise
    end
    FileUtils.mv "#{cachefile}.partial", cachefile
    true
end

#download_http(package, uri, filename, user: nil, password: nil, current_time: nil) ⇒ Object

rubocop:disable Metrics/ParameterLists



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
 
# File 'lib/autobuild/import/archive.rb', line 107

def download_http(package, uri, filename, # rubocop:disable Metrics/ParameterLists
        user: nil, password: nil, current_time: nil)
    request = Net::HTTP::Get.new(uri)
    request['If-Modified-Since'] = current_time.rfc2822 if current_time
    request.basic_auth(user, password) if user

    Net::HTTP.start(
        uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
        http.request(request) do |resp|
            case resp
            when Net::HTTPNotModified
                return false
            when Net::HTTPSuccess
                if current_time && (last_modified = resp['last-modified']) &&
                   (current_time >= Time.rfc2822(last_modified))
                   return false
                end

                if (length = resp['Content-Length'])
                    length = Integer(length)
                    expected_size = "/#{Autobuild.human_readable_size(length)}"
                end

                File.open(filename, 'wb') do |io|
                    size = 0
                    next_update = Time.now
                    resp.read_body do |chunk|
                        io.write chunk
                        size += chunk.size
                        if size != 0 && (Time.now > next_update)
                            formatted_size = Autobuild.human_readable_size(size)
                            package.progress "downloading %s "\
                                "(#{formatted_size}#{expected_size})"
                            next_update = Time.now + 1
                        end
                    end
                    formatted_size = Autobuild.human_readable_size(size)
                    package.progress "downloaded %s "\
                        "(#{formatted_size}#{expected_size})"
                end
            when Net::HTTPRedirection
                if (location = resp['location']).start_with?('/')
                    redirect_uri = uri.dup
                    redirect_uri.path = resp['location']
                else
                    redirect_uri = location
                end

                return download_http(package, URI(redirect_uri), filename,
                                     user: user, password: password,
                                     current_time: current_time)
            else
                raise PackageException.new(package, 'import'),
                      "failed download of #{package.name} from #{uri}: "\
                      "#{resp.class}"
            end
        end
    end
    true
end

#extract_tar_gz(io, target) ⇒ Object 



168
169
170
171
172
173
174
175
176
177
178
179
180
181
 
# File 'lib/autobuild/import/archive.rb', line 168

def extract_tar_gz(io, target)
    Gem::Package::TarReader.new(io).each do |entry|
        newname = File.join(
            target, File.basename(entry.full_name))
        FileUtils.mkdir_p(newname) if entry.directory?
        if entry.file?
            dir = File.dirname(newname)
            FileUtils.mkdir_p(dir) unless File.directory?(dir)
            File.open(newname, "wb") do |file|
                file.write(entry.read)
            end
        end
    end
end

#has_subdirectory? ⇒ Boolean

Tests whether the archive’s content is stored within a subdirectory or not

If it has a subdirectory, its name is assumed to be the package’s basename, or the value returned by #archive_dir if the archive_dir option was given to #initialize

Returns:

  • (Boolean) 


361
362
363
 
# File 'lib/autobuild/import/archive.rb', line 361

def has_subdirectory?
    !@options[:no_subdirectory]
end

#read_cachefile_digest ⇒ Object 



277
278
279
 
# File 'lib/autobuild/import/archive.rb', line 277

def read_cachefile_digest
    Digest::SHA1.hexdigest File.read(cachefile)
end

#relocate(url, options = Hash.new) ⇒ Object

Changes the URL from which we should pick the archive



397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
 
# File 'lib/autobuild/import/archive.rb', line 397

def relocate(url, options = Hash.new)
    parsed_url = URI.parse(url).normalize
    @url = parsed_url
    if !VALID_URI_SCHEMES.include?(@url.scheme)
        raise ConfigException, "invalid URL #{@url} (local files "\
            "must be prefixed with file://)"
    elsif Autobuild.windows?
        unless WINDOWS_VALID_URI_SCHEMES.include?(@url.scheme)
            raise ConfigException, "downloading from a #{@url.scheme} URL "\
                "is not supported on windows"
        end
    end

    @repository_id = options[:repository_id] || parsed_url.to_s
    @source_id     = options[:source_id] || parsed_url.to_s
    @expected_digest = options[:expected_digest]

    @filename =
        options[:filename] ||
        @filename ||
        File.basename(url).gsub(/\?.*/, '')
    @update_cached_file = options[:update_cached_file]

    @mode =
        options[:mode] ||
        ArchiveImporter.find_mode_from_filename(filename) ||
        @mode

    if Autobuild.windows? && (mode != Gzip)
        raise ConfigException, "only gzipped tar archives "\
            "are supported on Windows"
    end
    @user = options[:user]
    @password = options[:password]
    if @user && !%w[http https].include?(@url.scheme)
        raise ConfigException, "authentication is only supported for "\
            "http and https URIs"
    end

    @cachefile =
        if @url.scheme == 'file'
            @url.path
        else
            File.join(cachedir, filename)
        end
end

#tardir ⇒ Object

Deprecated.

use #archive_dir instead



321
322
323
 
# File 'lib/autobuild/import/archive.rb', line 321

def tardir
    @options[:tardir]
end

#update(package, options = Hash.new) ⇒ Object

:nodoc:



444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
 
# File 'lib/autobuild/import/archive.rb', line 444

def update(package, options = Hash.new) # :nodoc:
    if options[:only_local]
        package.warn "%s: the archive importer does not support local updates, "\
            "skipping"
        return false
    end
    needs_update = update_cache(package)

    unless File.file?(checkout_digest_stamp(package))
        write_checkout_digest_stamp(package)
    end

    if needs_update || archive_changed?(package)
        checkout(package, allow_interactive: options[:allow_interactive])
    else
        false
    end
end

#update_cache(package) ⇒ Boolean

Updates the downloaded file in cache only if it is needed

Returns:

  • (Boolean) —

    true if a new file was downloaded, false otherwise



265
266
267
268
269
270
271
272
273
274
275
 
# File 'lib/autobuild/import/archive.rb', line 265

def update_cache(package)
    updated = download_from_url(package)
    @cachefile_digest = read_cachefile_digest

    if @expected_digest && @expected_digest != @cachefile_digest
        raise ConfigException,
              "The archive #{@url} does not match the digest provided"
    end

    updated
end

#update_cached_file? ⇒ Boolean

Returns:

  • (Boolean) 


103
104
105
 
# File 'lib/autobuild/import/archive.rb', line 103

def update_cached_file?
    @update_cached_file
end

#update_needed?(package) ⇒ Boolean

Returns:

  • (Boolean) 


183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
 
# File 'lib/autobuild/import/archive.rb', line 183

def update_needed?(package)
    return true  unless File.file?(cachefile)
    return false unless update_cached_file?

    cached_size = File.lstat(cachefile).size
    cached_mtime = File.lstat(cachefile).mtime

    size, mtime = nil
    if @url.scheme == "file"
        size  = File.stat(@url.path).size
        mtime = File.stat(@url.path).mtime
    else
        # rubocop:disable Security/Open
        open @url, :content_length_proc => ->(v) { size = v } do |file|
            mtime = file.last_modified
        end
        # rubocop:enable Security/Open
    end

    if mtime && size
        size != cached_size || mtime > cached_mtime
    elsif mtime
        package.warn "%s: archive size is not available for #{@url}, "\
            "relying on modification time"
        mtime > cached_mtime
    elsif size
        package.warn "%s: archive modification time "\
            "is not available for #{@url}, relying on size"
        size != cached_size
    else
        package.warn "%s: neither the archive size nor its modification time "\
            "are available for #{@url}, will always update"
        true
    end
end

#vcs_fingerprint(_package) ⇒ Object

Fingerprint for archive importer, we are using its digest whether is calculated or expected



284
285
286
287
288
289
290
291
292
293
294
295
296
 
# File 'lib/autobuild/import/archive.rb', line 284

def vcs_fingerprint(_package)
    if @cachefile_digest
        @cachefile_digest
    elsif File.file?(cachefile)
        read_cachefile_digest
    elsif @expected_digest
        @expected_digest
    else
        raise ConfigException,
              "There is no digest for archive #{@url}, make sure "\
              "cache directories are configured correctly"
    end
end

#write_checkout_digest_stamp(package) ⇒ Object 



467
468
469
470
471
 
# File 'lib/autobuild/import/archive.rb', line 467

def write_checkout_digest_stamp(package)
    File.open(checkout_digest_stamp(package), 'w') do |io|
        io.write cachefile_digest
    end
end