Class: Botz::Definition

Inherits:
Object
  • Object
show all
Defined in:
lib/botz/macro.rb

Overview

Class representing a website defined by DSL

Constant Summary collapse

Output = 
->(result) { STDOUT.puts(result.to_json) }

Class Method Summary collapse

Class Method Details

.before_context(url:) ⇒ Object 



55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/botz/macro.rb', line 55

def before_context(url:)
  downloader = Botz::Downloader.new(:html)
  before_context_class = Class.new do
    define_singleton_method(:call) do
      downloader.call(url) do |page|
        yield(page)
        page.mech
      end
    end
  end
  const_set('before_context'.classify, before_context_class)
end

.output(&block) ⇒ Object 



16
17
18
19
 
# File 'lib/botz/macro.rb', line 16

def output(&block)
  remove_const(:Output)
  const_set(:Output, block)
end

.scraper(name, as: :html, output: Output, &block) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/botz/macro.rb', line 21

def scraper(name, as: :html, output: Output, &block)
  class_name = "#{name}_scraper".classify
  downloader = Botz::Downloader.new(as)
  binder_base = Botz.const_get "#{as}_scraper_macro".classify
  binder = Class.new(binder_base, &block)
  binder.define_singleton_method(:name) { class_name }
  crawler_class = self
  scraper_class = Class.new do
    define_singleton_method(:crawler_class) { crawler_class }
    define_singleton_method(:bind) do |url|
      downloader.call(url) do |resource|
        binder.new(scraper_class, resource, output)
      end
    end
    define_singleton_method(:call) { |url| bind(url).save }
  end
  const_set(class_name, scraper_class)
  scrapers[name] = scraper_class
end

.spider(name, start_url = nil, as: :html, &block) ⇒ Object 



41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/botz/macro.rb', line 41

def spider(name, start_url = nil, as: :html, &block)
  downloader = Botz::Downloader.new(as)
  spider = Botz::Spider.new(&block)
  spider_class = Class.new do
    define_singleton_method(:call) do |url = start_url, &spider_block|
      downloader.call(url) do |resource|
        spider.call(resource, &spider_block)
      end
    end
  end
  const_set("#{name}_spider".classify, spider_class)
  spiders[name] = spider_class
end