Class: Fetcher

Inherits:
Object
  • Object
show all
Defined in:
lib/sape/fetcher.rb

Class Method Summary collapse

Class Method Details

.fetch_config(config_data, bot_ips, site_host) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/sape/fetcher.rb', line 47

def fetch_config(config_data, bot_ips, site_host)
  SapeConfig.where(site_host: site_host).delete_all
  say "Ips:"
  bot_ips.each do |ip|
    SapeConfig.create name: 'ip', value: ip
    say "   Added #{ip}"
  end
  say "Config"
  config_data.each do |item, data|
    SapeConfig.create site_host: site_host, name: item, value: data
    say "   Added #{item} = #{data}"
  end
end

.fetch_pages(pages, link_type, delete_old = false, site_host) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/sape/fetcher.rb', line 30

def fetch_pages(pages, link_type, delete_old = false, site_host)
  SapeLink.where(site_host: site_host).delete_all if delete_old
  say "Links:: #{link_type}"
  pages.each do |page_url, links|
    say "Page: #{page_url}"
    links.each do |link|
      item = Nokogiri::HTML.parse(link)

      anchor    = item.css('a').text
      url       = item.css('a').attr('href').text
      url_host  = Domainatrix.parse(url).host
      SapeLink.create site_host: site_host, page: page_url, anchor: anchor, host: url_host, raw_link: link, url: url, link_type: link_type
      say "   Added #{site_host} :: #{anchor} #{url_host} #{url}"
    end unless links.nil?
  end
end

.get_data(config, link_type, site_host) ⇒ Object 



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/sape/fetcher.rb', line 12

def get_data(config, link_type, site_host)

  sape_user  = config['sape_user']
  charset    = config['charset']       || 'utf-8'
  server     = config['server']        || 'dispenser-01.sape.ru'
  links_type = {'simple' => 'code.php', 'context' => 'code_context.php'}

  url = "http://#{server}/#{links_type[link_type]}?user=#{sape_user}&host=#{site_host}&format=json&no_slash_fix=true"

  begin
    data = open(url)
  rescue OpenURI::HTTPError
    fail "Could not receive data"
  end

  JSON.parse(data.read)
end

.say(text) ⇒ Object 



8
9
10
 
# File 'lib/sape/fetcher.rb', line 8

def say text
  puts text if Rails.env.development?
end