Class: Fetchers::Url

Inherits:
Object
  • Object
show all
Defined in:
lib/fetchers/url.rb

Direct Known Subclasses

Compliance::Fetcher, Supermarket::Fetcher

Constant Summary collapse

MIME_TYPES = 
{
  'application/x-zip-compressed' => '.zip',
  'application/zip' => '.zip',
  'application/x-gzip' => '.tar.gz',
  'application/gzip' => '.tar.gz',
}.freeze

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(url, opts) ⇒ Url

Returns a new instance of Url.



100
101
102
103
 
# File 'lib/fetchers/url.rb', line 100

def initialize(url, opts)
  @target = url
  @archive = self.class.download_archive(url, opts)
end

Instance Attribute Details

#archive ⇒ Object (readonly)

Returns the value of attribute archive.



98
99
100
 
# File 'lib/fetchers/url.rb', line 98

def archive
  @archive
end

#files ⇒ Object (readonly)

Returns the value of attribute files.



14
15
16
 
# File 'lib/fetchers/url.rb', line 14

def files
  @files
end

Class Method Details

.download_archive(url, opts = {}) ⇒ Object

download url into archive using opts, returns File object and content-type from HTTP headers



68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/fetchers/url.rb', line 68

def self.download_archive(url, opts = {})
  http_opts = {}
  # http_opts['http_basic_authentication'] = [opts['user'] || '', opts['password'] || ''] if opts['user']
  http_opts['ssl_verify_mode'.to_sym] = OpenSSL::SSL::VERIFY_NONE if opts['insecure']
  http_opts['Authorization'] = "Bearer #{opts['token']}" if opts['token']

  remote = open(
    url,
    http_opts,
  )

  content_type = remote.meta['content-type']
  file_type = MIME_TYPES[content_type] ||
              throw(RuntimeError, 'Failed to resolve URL target, its '\
              "metadata did not match ZIP or TAR: #{content_type}")

  # fall back to tar
  if file_type.nil?
    fail "Could not determine file type for content type #{content_type}."
  end

  # download content
  archive = Tempfile.new(['inspec-dl-', file_type])
  archive.binmode
  archive.write(remote.read)
  archive.rewind
  archive.close
  archive
end

.resolve(target, opts = {}) ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/fetchers/url.rb', line 16

def self.resolve(target, opts = {})
  uri = URI.parse(target)
  return nil if uri.nil? or uri.scheme.nil?
  return nil unless %{ http https }.include? uri.scheme
  target = transform(target)
  # fetch this url and hand it off
  res = new(target, opts)
  resolve_next(res.archive.path, res)
rescue URI::Error => _e
  nil
end

.transform(target) ⇒ Object

Transforms a browser github url to github tar url We distinguish between three different Github URL types:

- Master URL
- Branch URL
- Commit URL

master url: github.com/nathenharvey/tmp_compliance_profile/ is transformed to github.com/nathenharvey/tmp_compliance_profile/archive/master.tar.gz

github branch: github.com/hardening-io/tests-os-hardening/tree/2.0 is transformed to github.com/hardening-io/tests-os-hardening/archive/2.0.tar.gz

github commit: github.com/hardening-io/tests-os-hardening/tree/48bd4388ddffde68badd83aefa654e7af3231876 is transformed to github.com/hardening-io/tests-os-hardening/archive/48bd4388ddffde68badd83aefa654e7af3231876.tar.gz



46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/fetchers/url.rb', line 46

def self.transform(target)
  # support for default github url
  m = %r{^https?://(www\.)?github\.com/(?<user>[\w-]+)/(?<repo>[\w-]+)(\.git)?(/)?$}.match(target)
  return "https://github.com/#{m[:user]}/#{m[:repo]}/archive/master.tar.gz" if m

  # support for branch and commit urls
  m = %r{^https?://(www\.)?github\.com/(?<user>[\w-]+)/(?<repo>[\w-]+)/tree/(?<commit>[\w\.]+)(/)?$}.match(target)
  return "https://github.com/#{m[:user]}/#{m[:repo]}/archive/#{m[:commit]}.tar.gz" if m

  # if we could not find a match, return the original value
  target
end