Class: NVDFeedScraper::Feed

Inherits:
Object
  • Object
show all
Defined in:
lib/nvd_feed_api/feed.rb

Overview

Feed object.

Class Attribute Summary collapse

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(name, updated, meta_url, gz_url, zip_url) ⇒ Feed

A new instance of Feed.

Parameters:



100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/nvd_feed_api/feed.rb', line 100

def initialize(name, updated, meta_url, gz_url, zip_url)
  # From meta file
  @name = name
  @updated = updated
  @meta_url = meta_url
  @gz_url = gz_url
  @zip_url = zip_url
  # do not pull meta and json automatically for speed and memory footprint
  @meta = nil
  @json_file = nil
  # feed data
  @data_type = nil
  @data_format = nil
  @data_version = nil
  @data_number_of_cves = nil
  @data_timestamp = nil
end

Class Attribute Details

.default_storage_location ⇒ String

Get / set default feed storage location, where will be stored JSON feeds and archives by default.

Examples:

NVDFeedScraper::Feed.default_storage_location = '/srv/downloads/'

Returns:

  • (String) —

    default feed storage location. Default to /tmp/.



21
22
23
 
# File 'lib/nvd_feed_api/feed.rb', line 21

def default_storage_location
  @default_storage_location
end

Instance Attribute Details

#data_format ⇒ String (readonly)

Note:

Return nil if not previously loaded by #json_pull.

Returns the format of the feed, should always be MITRE.

Returns:

  • (String) —

    the format of the feed, should always be MITRE.



80
81
82
 
# File 'lib/nvd_feed_api/feed.rb', line 80

def data_format
  @data_format
end

#data_number_of_cves ⇒ Integer (readonly)

Note:

Return nil if not previously loaded by #json_pull.

Returns the number of CVEs of in the feed.

Returns:

  • (Integer) —

    the number of CVEs of in the feed.



88
89
90
 
# File 'lib/nvd_feed_api/feed.rb', line 88

def data_number_of_cves
  @data_number_of_cves
end

#data_timestamp ⇒ Date (readonly)

Note:

Return nil if not previously loaded by #json_pull.

Returns the date of the last update of the feed by the NVD.

Returns:

  • (Date) —

    the date of the last update of the feed by the NVD.



92
93
94
 
# File 'lib/nvd_feed_api/feed.rb', line 92

def data_timestamp
  @data_timestamp
end

#data_type ⇒ String (readonly)

Note:

Return nil if not previously loaded by #json_pull.

Returns the type of the feed, should always be CVE.

Returns:

  • (String) —

    the type of the feed, should always be CVE.



76
77
78
 
# File 'lib/nvd_feed_api/feed.rb', line 76

def data_type
  @data_type
end

#data_version ⇒ Float (readonly)

Note:

Return nil if not previously loaded by #json_pull.

Returns the version of the JSON schema of the feed.

Returns:

  • (Float) —

    the version of the JSON schema of the feed.



84
85
86
 
# File 'lib/nvd_feed_api/feed.rb', line 84

def data_version
  @data_version
end

#gz_url ⇒ String

Returns the URL of the gz archive of the feed.

Examples:

'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2007.json.gz'

Returns:

  • (String) —

    the URL of the gz archive of the feed.



43
44
45
 
# File 'lib/nvd_feed_api/feed.rb', line 43

def gz_url
  @gz_url
end

#json_file ⇒ String (readonly)

Note:

Return nil if not previously loaded by #json_pull.

Returns the path of the saved JSON file.

Examples:

s = NVDFeedScraper.new
s.scrap
f = s.feeds("CVE-2014")
f.json_file # => nil
f.json_pull
f.json_file # => "/tmp/nvdcve-1.0-2014.json"

Returns:

  • (String) —

    the path of the saved JSON file.



72
73
74
 
# File 'lib/nvd_feed_api/feed.rb', line 72

def json_file
  @json_file
end

#meta ⇒ Meta (readonly)

Note:

Return nil if not previously loaded by #meta_pull. Note that #json_pull also calls #meta_pull.

Returns the Meta object of the feed.

Examples:

s = NVDFeedScraper.new
s.scrap
f = s.feeds("CVE-2014")
f.meta # => nil
f.meta_pull
f.meta # => #<NVDFeedScraper::Meta:0x00555b53027570 ... >

Returns:

  • (Meta) —

    the Meta object of the feed.



61
62
63
 
# File 'lib/nvd_feed_api/feed.rb', line 61

def meta
  @meta
end

#meta_url ⇒ String

Returns the URL of the metadata file of the feed.

Examples:

'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2007.meta'

Returns:

  • (String) —

    the URL of the metadata file of the feed.



38
39
40
 
# File 'lib/nvd_feed_api/feed.rb', line 38

def meta_url
  @meta_url
end

#name ⇒ String

Returns the name of the feed.

Examples:

'CVE-2007'

Returns:

  • (String) —

    the name of the feed.



28
29
30
 
# File 'lib/nvd_feed_api/feed.rb', line 28

def name
  @name
end

#updated ⇒ String

Returns the last update date of the feed information on the NVD website.

Examples:

'10/19/2017 3:27:02 AM -04:00'

Returns:

  • (String) —

    the last update date of the feed information on the NVD website.



33
34
35
 
# File 'lib/nvd_feed_api/feed.rb', line 33

def updated
  @updated
end

#zip_url ⇒ String

Returns the URL of the zip archive of the feed.

Examples:

'https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-2007.json.zip'

Returns:

  • (String) —

    the URL of the zip archive of the feed.



48
49
50
 
# File 'lib/nvd_feed_api/feed.rb', line 48

def zip_url
  @zip_url
end

Instance Method Details

#available_cves ⇒ Array<String>

Return a list with the name of all available CVEs in the feed. Can only be called after #json_pull.

Returns:

  • (Array<String>) —

    List with the name of all available CVEs. May return thousands CVEs.



280
281
282
283
284
285
286
287
288
289
290
291
292
293
 
# File 'lib/nvd_feed_api/feed.rb', line 280

def available_cves
  raise 'json_file is nil, it needs to be populated with json_pull' if @json_file.nil?
  raise "json_file (#{@json_file}) doesn't exist" unless File.file?(@json_file)

  doc = Oj::Doc.open(File.read(@json_file))
  # Quicker than doc.fetch('/vulnerabilities').size
  cve_names = []
  (1..@data_number_of_cves).each do |i|
    doc.move("/vulnerabilities/#{i}")
    cve_names.push(doc.fetch('cve/id'))
  end
  doc.close
  return cve_names
end

#cve(cve) ⇒ Hash #cve(cve_arr) ⇒ Array #cve(cve, *) ⇒ Array

TODO:

implement a CVE Class instead of returning a Hash.

Note:

#json_pull is needed before using this method. Remember you're searching only in the current feed.

Search for CVE in the feed.

Examples:

s = NVDFeedScraper.new
s.scrap
f = s.feeds("CVE-2014")
f.json_pull
f.cve("CVE-2014-0002", "cve-2014-0001")

Overloads:

  • #cve(cve) ⇒ Hash

    One CVE.

    Parameters:

    • cve (String) —

      CVE ID, case insensitive.

    Returns:

    • (Hash) —

      a Ruby Hash corresponding to the CVE.

  • #cve(cve_arr) ⇒ Array

    An array of CVEs.

    Parameters:

    • cve_arr (Array<String>) —

      Array of CVE ID, case insensitive.

    Returns:

    • (Array) —

      an Array of CVE, each CVE is a Ruby Hash. May not be in the same order as provided.

  • #cve(cve, *) ⇒ Array

    Multiple CVEs.

    Parameters:

    • cve (String) —

      CVE ID, case insensitive.

    • * (String) —

      As many CVE ID as you want.

    Returns:

    • (Array) —

      an Array of CVE, each CVE is a Ruby Hash. May not be in the same order as provided.

See Also:



225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
 
# File 'lib/nvd_feed_api/feed.rb', line 225

def cve(*arg_cve)
  raise 'json_file is nil, it needs to be populated with json_pull' if @json_file.nil?
  raise "json_file (#{@json_file}) doesn't exist" unless File.file?(@json_file)

  return_value = nil
  raise 'no argument provided, 1 or more expected' if arg_cve.empty?

  if arg_cve.length == 1
    case arg_cve[0]
    when String
      raise "bad CVE name (#{arg_cve[0]})" unless /^CVE-[0-9]{4}-[0-9]{4,}$/i.match?(arg_cve[0])

      doc = Oj::Doc.open(File.read(@json_file))
      # Quicker than doc.fetch('/vulnerabilities').size
      (1..@data_number_of_cves).each do |i|
        if arg_cve[0].upcase == doc.fetch("/vulnerabilities/#{i}/cve/id")
          return_value = doc.fetch("/vulnerabilities/#{i}")
          break
        end
      end
      doc.close
    when Array
      return_value = []
      # Sorting CVE can allow us to parse quicker
      # Upcase to be sure include? works
      cves_to_find = arg_cve[0].map(&:upcase).sort
      raise 'one of the provided arguments is not a String' unless cves_to_find.all? { |x| x.is_a?(String) }
      raise 'bad CVE name' unless cves_to_find.all? { |x| /^CVE-[0-9]{4}-[0-9]{4,}$/i.match?(x) }

      doc = Oj::Doc.open(File.read(@json_file))
      # Quicker than doc.fetch('/vulnerabilities').size
      (1..@data_number_of_cves).each do |i|
        doc.move("/vulnerabilities/#{i}")
        cve_id = doc.fetch('cve/id')
        if cves_to_find.include?(cve_id)
          return_value.push(doc.fetch)
          cves_to_find.delete(cve_id)
        elsif cves_to_find.empty?
          break
        end
      end
      raise "#{cves_to_find.join(', ')} are unexisting CVEs in this feed" unless cves_to_find.empty?
    else
      raise "the provided argument (#{arg_cve[0]}) is nor a String or an Array"
    end
  else
    # Overloading a list of arguments as one array argument
    return_value = cve(arg_cve)
  end
  return return_value
end

#download_file(file_url, opts = {}) ⇒ String (protected)

Download a file.

Examples:

download_file('https://example.org/example.zip') # => '/tmp/example.zip'
download_file('https://example.org/example.zip', destination_path: '/srv/save/') # => '/srv/save/example.zip'
download_file('https://example.org/example.zip', {destination_path: '/srv/save/', sha256: '70d6ea136d5036b6ce771921a949357216866c6442f44cea8497f0528c54642d'}) # => '/srv/save/example.zip'

Parameters:

  • file_url (String) —

    the URL of the file.

  • opts (Hash) (defaults to: {}) —

    the optional downlaod parameters.

Options Hash (opts):

  • :destination_path (String) —

    the destination path (may overwrite existing file). Default use default_storage_location.

  • :sha256 (String) —

    the SHA256 hash to check, if the file already exist and the hash matches then the download will be skipped.

Returns:

  • (String) —

    the saved file path.



358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
 
# File 'lib/nvd_feed_api/feed.rb', line 358

def download_file(file_url, opts = {})
  opts[:destination_path] ||= Feed.default_storage_location
  opts[:sha256] ||= nil

  destination_path = opts[:destination_path]
  destination_path += '/' unless destination_path[-1] == '/'
  skip_download = false
  uri = URI(file_url)
  filename = uri.path.split('/').last
  destination_file = destination_path + filename
  if !opts[:sha256].nil? && File.file?(destination_file)
    # Verify hash to see if it is the latest
    computed_h = Digest::SHA256.file(destination_file)
    skip_download = true if opts[:sha256].casecmp(computed_h.hexdigest).zero?
  end
  unless skip_download
    res = Net::HTTP.get_response(uri)
    raise "#{file_url} ended with #{res.code} #{res.message}" unless res.is_a?(Net::HTTPSuccess)

    File.binwrite(destination_file, res.body)
  end
  return destination_file
end

#download_gz(opts = {}) ⇒ String

Download the gz archive of the feed.

Examples:

afeed.download_gz
afeed.download_gz(destination_path: '/srv/save/')

Parameters:

Returns:

  • (String) —

    the saved gz file path.



134
135
136
 
# File 'lib/nvd_feed_api/feed.rb', line 134

def download_gz(opts = {})
  download_file(@gz_url, opts)
end

#download_zip(opts = {}) ⇒ String

Download the zip archive of the feed.

Examples:

afeed.download_zip
afeed.download_zip(destination_path: '/srv/save/')

Parameters:

Returns:

  • (String) —

    the saved zip file path.



144
145
146
 
# File 'lib/nvd_feed_api/feed.rb', line 144

def download_zip(opts = {})
  download_file(@zip_url, opts)
end

#json_pull(opts = {}) ⇒ String

Note:

Will download and save the zip of the JSON file, unzip and save it. This massively consume time.

Download the JSON feed and fill the attribute.

Parameters:

Returns:

See Also:



153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
 
# File 'lib/nvd_feed_api/feed.rb', line 153

def json_pull(opts = {})
  opts[:destination_path] ||= Feed.default_storage_location

  skip_download = false
  destination_path = opts[:destination_path]
  destination_path += '/' unless destination_path[-1] == '/'
  filename = URI(@zip_url).path.split('/').last.chomp('.zip')
  # do not use @json_file for destination_file because of offline loading
  destination_file = destination_path + filename
  meta_pull
  if File.file?(destination_file)
    # Verify hash to see if it is the latest
    computed_h = Digest::SHA256.file(destination_file)
    skip_download = true if meta.sha256.casecmp(computed_h.hexdigest).zero?
  end
  if skip_download
    @json_file = destination_file
    # Set data
    if @data_type.nil?
      doc = Oj::Doc.open(File.read(@json_file))
      @data_type = 'CVE' # @data_type was doc.fetch('/CVE_data_type') in 1.1 schema but was removed in 2.0 so keep this string for retrocompatibility
      @data_format = doc.fetch('/format')
      @data_version = doc.fetch('/version').to_f
      @data_number_of_cves = doc.fetch('/totalResults').to_i
      @data_timestamp = Date.strptime(doc.fetch('/timestamp'), '%FT%T.%N')
      doc.close
    end
  else
    zip_path = download_zip(opts)
    Archive::Zip.open(zip_path) do |z|
      z.extract(destination_path, flatten: true)
    end
    @json_file = zip_path.chomp('.zip')
    # Verify hash integrity
    computed_h = Digest::SHA256.file(@json_file)
    raise "File corruption: #{@json_file}" unless meta.sha256.casecmp(computed_h.hexdigest).zero?

    # update data
    doc = Oj::Doc.open(File.read(@json_file))
    @data_type = 'CVE' # @data_type was doc.fetch('/CVE_data_type') in 1.1 schema but was removed in 2.0 so keep this string for retrocompatibility
    @data_format = doc.fetch('/format')
    @data_version = doc.fetch('/version').to_f
    @data_number_of_cves = doc.fetch('/totalResults').to_i
    @data_timestamp = Date.strptime(doc.fetch('/timestamp'), '%FT%T.%N')
    doc.close
  end
  return @json_file
end

#meta_pull ⇒ Meta

Create or update the Meta object (fill the attribute).

Returns:

  • (Meta) —

    the updated Meta object of the feed.

See Also:



121
122
123
124
125
126
 
# File 'lib/nvd_feed_api/feed.rb', line 121

def meta_pull
  meta_content = NVDFeedScraper::Meta.new(@meta_url)
  meta_content.parse
  # update @meta
  @meta = meta_content
end

#update!(fresh_feed) ⇒ Boolean

Note:

Is not intended to be used directly, use NVDFeedScraper#update_feeds instead.

Update the feed

Parameters:

  • fresh_feed (Feed) —

    the fresh feed from which the feed will be updated.

Returns:

  • (Boolean) —

    true if the feed was updated, false if it wasn't.



386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
 
# File 'lib/nvd_feed_api/feed.rb', line 386

def update!(fresh_feed)
  return_value = false
  raise "#{fresh_feed} is not a Feed" unless fresh_feed.is_a?(Feed)

  # update attributes
  if updated != fresh_feed.updated
    self.name = fresh_feed.name
    self.updated = fresh_feed.updated
    self.meta_url = fresh_feed.meta_url
    self.gz_url = fresh_feed.gz_url
    self.zip_url = fresh_feed.zip_url
    # update if @meta was set
    meta_pull unless @meta.nil?
    # update if @json_file was set, this will also update @data_*
    json_pull unless @json_file.nil?
    return_value = true
  end
  return return_value
end