Class: Blinkr::Engine

Inherits:
Object
  • Object
show all
Includes:
HttpUtils, Sitemap
Defined in:
lib/blinkr/engine.rb

Defined Under Namespace

Classes: ErrorArray

Instance Method Summary collapse

Methods included from Sitemap

#sitemap_locations

Methods included from HttpUtils

#retry?, #sanitize

Constructor Details

#initialize(config) ⇒ Engine 



30
31
32
33
34
 
# File 'lib/blinkr/engine.rb', line 30

def initialize config
  @config = config.validate
  @extensions = []
  load_pipeline
end

Instance Method Details

#analyze(context, typhoeus) ⇒ Object 



80
81
82
 
# File 'lib/blinkr/engine.rb', line 80

def analyze context, typhoeus
  exec :analyze, context, typhoeus
end

#append(context) ⇒ Object 



66
67
68
 
# File 'lib/blinkr/engine.rb', line 66

def append context
  exec :append, context
end

#collect(page) ⇒ Object 



84
85
86
 
# File 'lib/blinkr/engine.rb', line 84

def collect page
  exec :collect, page
end

#run ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/blinkr/engine.rb', line 36

def run
  context = OpenStruct.new({:pages => {}})
  typhoeus, browser = TyphoeusWrapper.new(@config, context)
  browser = PhantomJSWrapper.new(@config, context) if @config.browser == 'phantomjs'
  page_count = 0
  browser.process_all(sitemap_locations, @config.max_page_retrys) do |response, resource_errors, javascript_errors|
    if response.success?
      url = response.request.base_url
      puts "Loaded page #{url}" if @config.verbose
      body = Nokogiri::HTML(response.body)
      page = OpenStruct.new({ :response => response, :body => body, :errors => ErrorArray.new(@config), :resource_errors => resource_errors || [], :javascript_errors => javascript_errors || [] })
      context.pages[url] = page
      collect page
      page_count += 1
    else
      puts "#{response.code} #{response.status_message} Unable to load page #{url} #{'(' + response.return_message + ')' unless response.return_message.nil?}"
    end
  end
  typhoeus.hydra.run if @config.browser == 'typhoeus'
  analyze context, typhoeus
  puts "Loaded #{page_count} pages using #{browser.name}. Performed #{typhoeus.count} requests using typhoeus."
  context.pages.reject! { |url, page| page.errors.empty? }
  unless @config.export.nil?
    File.open(@config.export, 'w') do |file| 
      file.write(context.to_json)
    end
  end
  Blinkr::Report.new(context, self, @config).render
end

#transform(page, error, &block) ⇒ Object 



70
71
72
73
74
75
76
77
78
 
# File 'lib/blinkr/engine.rb', line 70

def transform page, error, &block
  default = yield
  result = exec(:transform, page, error, default)
  if result.empty?
    default
  else
    result.join
  end
end