Class: Retriever::FetchSitemap

Inherits:
Fetch
  • Object
show all
Defined in:
lib/retriever/fetchsitemap.rb

Instance Attribute Summary

Attributes inherited from Fetch

#max_pages, #t

Instance Method Summary collapse

Methods inherited from Fetch

#async_crawl_and_collect, #dump, #errlog, #good_response?, #lg, #process_link_stack, #write

Constructor Details

#initialize(url, options) ⇒ FetchSitemap

recieves target URL and RR options returns an array of all unique pages found on the site



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/retriever/fetchsitemap.rb', line 6

def initialize(url, options)
  super
  @data = [@t.target]
  page_one = Retriever::Page.new(@t.source, @t)
  lg("URL Crawled: #{@t.target}")
  @link_stack = page_one.parse_internal_visitable
  errlog("Bad URL -- #{@t.target}") unless @link_stack
  lg("#{@link_stack.size - 1} links found")

  @link_stack.delete(@t.target)
  @data.concat(@link_stack)

  async_crawl_and_collect

  @data.sort_by! { |x| x.length } if @data.size > 1
  @data.uniq!
end

Instance Method Details

#gen_xml ⇒ Object

produces valid XML sitemap based on page collection fetched. Writes to current directory.



26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/retriever/fetchsitemap.rb', line 26

def gen_xml
  f = File.open("sitemap-#{@t.host.split('.')[1]}.xml", 'w+')
  f << "<?xml version='1.0' encoding='UTF-8'?><urlset xmlns='http://www.sitemaps.org/schemas/sitemap/0.9'>"
    @data.each do |url|
      f << "<url><loc>#{url}</loc></url>"
    end
  f << '</urlset>'
  f.close
  puts '###############################'
  puts "File Created: sitemap-#{@t.host.split('.')[1]}.xml"
  puts "Object Count: #{@data.size}"
  puts '###############################'
  puts
end