Class: Pageflow::Chart::ScrapeSiteJob

Inherits:
Object
  • Object
show all
Extended by:
StateMachineJob
Defined in:
app/jobs/pageflow/chart/scrape_site_job.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(downloader) ⇒ ScrapeSiteJob 



9
10
11
 
# File 'app/jobs/pageflow/chart/scrape_site_job.rb', line 9

def initialize(downloader)
  @downloader = downloader
end

Instance Attribute Details

#downloader ⇒ Object (readonly)

Returns the value of attribute downloader.



7
8
9
 
# File 'app/jobs/pageflow/chart/scrape_site_job.rb', line 7

def downloader
  @downloader
end

Class Method Details

.perform_with_result(scraped_site, options = {}) ⇒ Object 



43
44
45
46
 
# File 'app/jobs/pageflow/chart/scrape_site_job.rb', line 43

def self.perform_with_result(scraped_site, options = {})
  # This is were the downloader passed to `initialize` is created.
  new(Downloader.new(base_url: scraped_site.url)).perform(scraped_site)
end

Instance Method Details

#begin_try_catch ⇒ Object 



48
49
50
 
# File 'app/jobs/pageflow/chart/scrape_site_job.rb', line 48

def begin_try_catch
  ";try {\n"
end

#end_try_catch ⇒ Object 



52
53
54
 
# File 'app/jobs/pageflow/chart/scrape_site_job.rb', line 52

def end_try_catch
  "\n} catch(e) { console.log('Datawrapper raised: ' + (e.message || e)); }\n"
end

#perform(scraped_site) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'app/jobs/pageflow/chart/scrape_site_job.rb', line 13

def perform(scraped_site)
  downloader.load(scraped_site.url) do |file|
    scraper = Scraper.new(file.read, Chart.config.scraper_options)
    scraped_site.html_file = StringIOWithContentType.new(
      scraper.html,
      file_name: 'file.html',
      content_type: 'text/html'
    )

    downloader.load_all(scraper.javascript_urls,
                        extension: '.js',
                        before_each: begin_try_catch,
                        after_each: end_try_catch) do |javascript_file|
      scraped_site.javascript_file = javascript_file
    end

    downloader.load_all(scraper.stylesheet_urls,
                        extension: '.css',
                        separator: "\n;") do |stylesheet_file|
      scraped_site.stylesheet_file = stylesheet_file
    end
  end

  downloader.load(scraped_site.csv_url) do |file|
    scraped_site.csv_file = file
  end

  :ok
end