Module: Crags::Searcher

Includes:
Fetch, ERB::Util
Included in:
Runner
Defined in:
lib/crags/searcher.rb

Instance Method Summary collapse

Methods included from Fetch

#fetch_doc, #fetch_html, #fetch_request

Instance Method Details

#categories ⇒ Object 



35
36
37
38
39
40
41
42
43
 
# File 'lib/crags/searcher.rb', line 35

def categories
  doc = fetch_doc("http://sfbay.craigslist.org/")
  links = doc.search("table[@summary=\"for sale\"] a")
  categories = {}
  links.each do |link|
    categories[link.inner_html] = link["href"]
  end
  categories
end

#hashify(item) ⇒ Object 



58
59
60
61
62
63
 
# File 'lib/crags/searcher.rb', line 58

def hashify(item)
  title = item.at("title").inner_text
  url = strip_http(item["rdf:about"])
  date = DateTime.parse(item.at("dc:date").inner_text)
  {:title => title, :url => url, :date => date}
end

#items(doc) ⇒ Object 



52
53
54
55
56
 
# File 'lib/crags/searcher.rb', line 52

def items(doc)
  doc.search("item").collect do |item|
    hashify(item)
  end
end

#location_doc(country) ⇒ Object 



14
15
16
 
# File 'lib/crags/searcher.rb', line 14

def location_doc(country)
  fetch_doc(location_link(country))
end

#location_link(country) ⇒ Object 



10
11
12
 
# File 'lib/crags/searcher.rb', line 10

def location_link(country)
  "http://geo.craigslist.org/iso/#{country}"
end

#location_links(country) ⇒ Object 



22
23
24
 
# File 'lib/crags/searcher.rb', line 22

def location_links(country)
  location_doc(country).search("#list a")
end

#location_request(country) ⇒ Object 



18
19
20
 
# File 'lib/crags/searcher.rb', line 18

def location_request(country)
  fetch_request(location_link(country))
end

#locations(country) ⇒ Object 



26
27
28
29
30
31
32
33
 
# File 'lib/crags/searcher.rb', line 26

def locations(country)
  linkz = location_links(country)
  if linkz.empty?
    [strip_http(location_request(country).last_effective_url)]
  else
    linkz.collect{|link| strip_http(link["href"]) }
  end
end

#search(keyword, country = 'us', category = 'sss', &block) ⇒ Object 



45
46
47
48
49
50
 
# File 'lib/crags/searcher.rb', line 45

def search(keyword, country = 'us', category = 'sss', &block)
  locations(country).collect do |loc|
    sleep(1 + rand(3))
    search_location(keyword, loc, category, &block)
  end.flatten
end

#search_location(keyword, loc, category = 'sss', &block) ⇒ Object 



69
70
71
72
73
74
75
 
# File 'lib/crags/searcher.rb', line 69

def search_location(keyword, loc, category = 'sss', &block)
  doc = fetch_doc("#{search_location_link(keyword, loc, category)}&format=rss")
  items(doc).collect do |item|
    yield item if block_given?
    item
  end
end

#search_location_link(keyword, loc, category = 'sss') ⇒ Object 



65
66
67
 
# File 'lib/crags/searcher.rb', line 65

def search_location_link(keyword, loc, category = 'sss')
  "http://#{loc}/search/#{category}?query=#{url_encode(keyword)}"
end

#strip_http(url) ⇒ Object 



6
7
8
 
# File 'lib/crags/searcher.rb', line 6

def strip_http(url)
  url.gsub(/^http\:\/\//,'').gsub(/\/$/,'')
end