Module: Spidy::Definition

Defined in:
lib/spidy/definition.rb

Overview

Class representing a website defined by DSL

Instance Method Summary collapse

Instance Method Details

#call(source = nil, name: :default, &yielder) ⇒ Object 



11
12
13
14
15
16
17
 
# File 'lib/spidy/definition.rb', line 11

def call(source = nil, name: :default, &yielder)
  name = name.presence || :default
  spidy = @namespace[:"#{name}_scraper"]
  fail "undefined spidy [#{name}]" if spidy.nil?

  spidy.call(source, &yielder)
end

#define(name = :default, connector: nil, as: nil, &define_block) ⇒ Object 



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/spidy/definition.rb', line 54

def define(name = :default, connector: nil, as: nil, &define_block)
  connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent,
                                                    socks_proxy: @socks_proxy)
  binder_base = Spidy::Binder.const_get(as.to_s.classify)
  @namespace ||= {}
  @namespace[:"#{name}_scraper"] = Class.new(Spidy::DefinitionObject) do
    extend binder_base
    class_eval(&define_block)
    define_singleton_method(:call) do |source, &yielder|
      yielder = ->(result) { break result } if yielder.nil?
      connection_yielder = lambda do |page|
        yielder.call(new(page, source))
      end
      connector.call(source, &connection_yielder)
    end
  end
end

#each(source = nil, name: :default, &yielder) ⇒ Object 



31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/spidy/definition.rb', line 31

def each(source = nil, name: :default, &yielder)
  name = name.presence || :default
  spidy = @namespace[:"#{name}_spider"]
  fail "undefined spidy [#{name}]" if spidy.nil?

  if yielder
    spidy.call(source, &yielder)
  else
    Enumerator.new do |enumerate_yielder|
      spidy.call(source, &enumerate_yielder)
    end
  end
end

#namespace ⇒ Object 



7
8
9
 
# File 'lib/spidy/definition.rb', line 7

def namespace
  @namespace
end

#socks_proxy(host, port) ⇒ Object 



27
28
29
 
# File 'lib/spidy/definition.rb', line 27

def socks_proxy(host, port)
  @socks_proxy = { host: host, port: port }
end

#spider(name = :default, connector: nil, as: nil, &define_block) ⇒ Object 



45
46
47
48
49
50
51
52
 
# File 'lib/spidy/definition.rb', line 45

def spider(name = :default, connector: nil, as: nil, &define_block)
  @namespace ||= {}
  connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent,
                                                    socks_proxy: @socks_proxy)
  @namespace[:"#{name}_spider"] = proc do |source, &yielder|
    define_block.call(yielder, connector, source)
  end
end

#user_agent(user_agent) ⇒ Object 



19
20
21
 
# File 'lib/spidy/definition.rb', line 19

def user_agent(user_agent)
  @user_agent = user_agent
end

#wait_time(wait_time) ⇒ Object 



23
24
25
 
# File 'lib/spidy/definition.rb', line 23

def wait_time(wait_time)
  @wait_time = wait_time
end