Class: WonderScrape::Recorder

Inherits:
Object
  • Object
show all
Defined in:
lib/wonder_scrape/recorder.rb

Instance Method Summary collapse

Constructor Details

#initialize(output, options = {}) ⇒ Recorder

Returns a new instance of Recorder.



6
7
8
9
10
11
12
13
 
# File 'lib/wonder_scrape/recorder.rb', line 6

def initialize(output, options = {})
  @output = output
  @verbose = options[:verbose] || false
  @progress_bar = options[:progress_bar]
  @items_scraped = 0
  @item_issues = {}
  @unexpected_fields = []
end

Instance Method Details

#increment_items_scraped(item) ⇒ Object 



28
29
30
31
32
33
34
35
 
# File 'lib/wonder_scrape/recorder.rb', line 28

def increment_items_scraped(item)
  @items_scraped += 1
  if verbose
    output.puts JSON.pretty_generate(item)
  else
    progress_bar&.advance(1)
  end
end

#print ⇒ Object 



15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/wonder_scrape/recorder.rb', line 15

def print
  output.puts "Successfully processed #{items_scraped} items!"

  if unexpected_fields.count > 0
    output.puts "Encountered the following unexpected fields: #{unexpected_fields}"
  end

  if item_issues.count > 0
    output.puts "Had issues with #{item_issues.count} items below"
    output.puts JSON.pretty_generate(item_issues)
  end
end

#record_unexpected_field(item_id, field_name) ⇒ Object 



37
38
39
40
41
 
# File 'lib/wonder_scrape/recorder.rb', line 37

def record_unexpected_field(item_id, field_name)
  item_issues[item_id] ||= []
  item_issues[item_id] << "Unexpected field: #{field_name}"
  unexpected_fields << field_name
end