Class: Retriever::Fetch

Inherits:
Object
  • Object
show all
Defined in:
lib/retriever/fetch.rb

Direct Known Subclasses

FetchFiles, FetchSEO, FetchSitemap

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(url, options) ⇒ Fetch

given target URL and RR options, creates a fetch object. There is no direct output this is a parent class that the other fetch classes build off of.



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/retriever/fetch.rb', line 16

def initialize(url, options)
  @connection_tally = {
    :success => 0,
    :error => 0,
    :error_client => 0,
    :error_server => 0
  }
  # OPTIONS
  @prgrss = options['progress']
  @max_pages = options['maxpages'] ? options['maxpages'].to_i : 100
  @v = options['verbose']
  @output = options['filename']
  @fh = options['fileharvest']
  @file_ext = @fh.to_s
  @s = options['sitemap']
  @seo = options['seo']
  @autodown = options['autodown']
  #
  if @fh
    temp_ext_str = '.' + @file_ext + '\z'
    @file_re = Regexp.new(temp_ext_str).freeze
  else
    # when FH is not true, and autodown is true
    errlog('Cannot AUTODOWNLOAD when not in FILEHARVEST MODE') if @autodown
  end
  if @prgrss
    # verbose & progressbar conflict
    errlog('CANNOT RUN VERBOSE & PROGRESSBAR AT SAME TIME, CHOOSE ONE, -v or -p') if @v
    prgress_vars = {
      :title => 'Pages',
      :starting_at => 1,
      :total => @max_pages,
      :format => '%a |%b>%i| %c/%C %t'
    }
    @progressbar = ProgressBar.create(prgress_vars)
  end
  @t = Retriever::Target.new(url, @file_re)
  @output = "rr-#{@t.host.split('.')[1]}" if @fh && !@output
  @already_crawled = BloomFilter::Native.new(
    :size => 1_000_000,
    :hashes => 5,
    :seed => 1,
    :bucket => 8,
    :raise => false
  )
  @already_crawled.insert(@t.target)
end

Instance Attribute Details

#max_pages ⇒ Object (readonly)

Returns the value of attribute max_pages.



12
13
14
 
# File 'lib/retriever/fetch.rb', line 12

def max_pages
  @max_pages
end

#t ⇒ Object (readonly)

Returns the value of attribute t.



12
13
14
 
# File 'lib/retriever/fetch.rb', line 12

def t
  @t
end

Instance Method Details

#async_crawl_and_collect ⇒ Object

iterates over the existing @link_stack running until we reach the @max_pages value.



123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
 
# File 'lib/retriever/fetch.rb', line 123

def async_crawl_and_collect
  while @already_crawled.size < @max_pages
    if @link_stack.empty?
      if @prgrss
        @progressbar.log("Can't find any more links.")
      else
        lg("Can't find any more links.")
      end
      break
    end
    new_links_arr = process_link_stack
    next if new_links_arr.nil? || new_links_arr.empty?
    # set operations to see are these in our previous visited pages arr
    new_links_arr -= @link_stack
    @link_stack.concat(new_links_arr).uniq!
    @data.concat(new_links_arr) if @s
  end
  # done, make sure progress bar says we are done
  @progressbar.finish if @prgrss
end

#dump ⇒ Object

prints current data collection to STDOUT



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/retriever/fetch.rb', line 73

def dump
  puts '###############################'
  if @v
    puts 'Connection Tally:'
    puts @connection_tally.to_s
    puts '###############################'
  end
  if @s
    puts "#{@t.target} Sitemap"
    puts "Page Count: #{@data.size}"
  elsif @fh
    puts "Target URL: #{@t.target}"
    puts "Filetype: #{@file_ext}"
    puts "File Count: #{@data.size}"
  elsif @seo
    puts "#{@t.target} SEO Metrics"
    puts "Page Count: #{@data.size}"
  else
    fail 'ERROR - Cannot dump - Mode Not Found'
  end
  puts '###############################'
  @data.each do |line|
    puts line
  end
  puts '###############################'
  puts
end

#errlog(msg) ⇒ Object 



64
65
66
 
# File 'lib/retriever/fetch.rb', line 64

def errlog(msg)
  fail "ERROR: #{msg}"
end

#good_response?(resp, url) ⇒ Boolean

returns true is resp is ok to continue

Returns:

  • (Boolean) 


145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
 
# File 'lib/retriever/fetch.rb', line 145

def good_response?(resp, url)
  return false unless resp
  hdr = resp.response_header
  if hdr.redirection?
    loc = hdr.location
    lg("#{url} Redirected to #{loc}")
    if t.host_re =~ loc
      @link_stack.push(loc) unless @already_crawled.include?(loc)
      lg('--Added to linkStack for later')
      return false
    end
    lg("Redirection outside of target host. No - go. #{loc}")
    return false
  end
  # lets not continue if unsuccessful connection
  unless hdr.successful?
    lg("UNSUCCESSFUL CONNECTION -- #{url}")

    @connection_tally[:error] += 1
    @connection_tally[:error_server] += 1 if hdr.server_error?
    @connection_tally[:error_client] += 1 if hdr.client_error?
    return false
  end
  # let's not continue if not text/html
  unless hdr['CONTENT_TYPE'].include?('text/html')
    @already_crawled.insert(url)
    @link_stack.delete(url)
    lg("Page Not text/html -- #{url}")
    return false
  end
  @connection_tally[:success] += 1
  true
end

#lg(msg) ⇒ Object 



68
69
70
 
# File 'lib/retriever/fetch.rb', line 68

def lg(msg)
  puts "### #{msg}" if @v
end

#process_link_stack ⇒ Object

send a new wave of GET requests, using current @link_stack



180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
 
# File 'lib/retriever/fetch.rb', line 180

def process_link_stack
  new_stuff = []
  EM.synchrony do
    concurrency = 10
    EM::Synchrony::FiberIterator.new(@link_stack, concurrency).each do |url|
      next if @already_crawled.size >= @max_pages
      next if @already_crawled.include?(url)

      resp = EventMachine::HttpRequest.new(url).get

      next unless good_response?(resp, url)
      lg("Page Fetched: #{url}")
      @already_crawled.insert(url)

      new_page = Retriever::Page.new(resp.response, @t)
      if @prgrss
        @progressbar.increment if @already_crawled.size < @max_pages
      end
      if @seo
        seos = [url]
        seos.concat(new_page.parse_seo)
        @data.push(seos)
        lg('--page SEO scraped')
      end
      next if new_page.links.size == 0
      lg("--#{new_page.links.size} links found")
      internal_links_arr = new_page.parse_internal_visitable
      new_stuff.push(internal_links_arr)
      if @fh
        filez = new_page.parse_files
        @data.concat(filez) unless filez.empty?
        lg("--#{filez.size} files found")
      end
    end
    new_stuff = new_stuff.flatten # all completed requests
    EventMachine.stop
  end
  new_stuff.uniq!
end

#write ⇒ Object

writes current data collection out to CSV in current directory



102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
 
# File 'lib/retriever/fetch.rb', line 102

def write
  return false unless @output
  i = 0
  CSV.open("#{@output}.csv", 'w') do |csv|
    if (i == 0) && @seo
      csv << ['URL', 'Page Title', 'Meta Description', 'H1', 'H2']
      i += 1
    end
    @data.each do |entry|
      csv << entry
    end
  end
  puts '###############################'
  puts "File Created: #{@output}.csv"
  puts "Object Count: #{@data.size}"
  puts '###############################'
  puts
end