Class: EchoLink::Scraper

Inherits:
Object
  • Object
show all
Defined in:
lib/echo_link/scraper.rb

Instance Method Summary collapse

Instance Method Details



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/echo_link/scraper.rb', line 8

def links
  links = []

  first_page_url = "#{EchoLink::URLS[:all_links]}&offset=0"
  html = html(first_page_url)
  links += links_from_table(html.css('table')[5])

  p = html.css('table').first.css('tr')[1].css('td').first.css('p').first
  total_links = p.css('b')[2].text.to_i
  per_page = 100
  page_count = (total_links / per_page.to_f).ceil

  (2..page_count).each do |num|
    offset = per_page * (num - 1)
    page_url = "#{EchoLink::URLS[:all_links]}&offset=#{offset}"

    html = html(page_url)
    links += links_from_table(html.css('table')[5])
  end
  links
end

#links_from_table(table) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/echo_link/scraper.rb', line 30

def links_from_table(table)
  links = []
  table.css('tr').each_with_index do |tr, i|
    next if i == 0
    tds = tr.css('td')
    links << {
      id: tds[2].text.to_i,
      call_sign: tds[0].text.gsub('\r\n',''),
      location: tds[3].text,
      grid: tds[4].text,
      frequency: tds[5].text,
      tone_squelch: tds[6].text,
      power: tds[7].text,
      haat: tds[8].text,
      antenna: tds[9].text,
      status: clean_status(tds[10].text),
      comment: tds[11].text,
      status_updated_at: tds[12].text
    }
  end
  links
end