Class: GitTrend::Scraper

Inherits:
Object
  • Object
show all
Defined in:
lib/git_trend/scraper.rb

Constant Summary collapse

BASE_HOST = 
'https://github.com'
BASE_URL = 
"#{BASE_HOST}/trending"

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Scraper

Returns a new instance of Scraper.



10
11
12
13
14
15
16
 
# File 'lib/git_trend/scraper.rb', line 10

def initialize
  @agent = Mechanize.new
  proxy = URI.parse(ENV['http_proxy']) if ENV['http_proxy']
  if proxy
    @agent.set_proxy(proxy.host, proxy.port, proxy.user, proxy.password)
  end
end

Instance Method Details

#get(language = nil, since = nil) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/git_trend/scraper.rb', line 18

def get(language=nil, since=nil)
  projects = []
  page = @agent.get(generate_url_for_get(language, since))

  page.search('.leaderboard-list-content').each do |content|
    project = Project.new
    project.lang        = content.search('.repo-leaderboard-title .title-meta').text
    project.name        = content.search('.repo-leaderboard-title a').text
    project.description = content.search('.repo-leaderboard-description').text

    project.star_count  = meta_count(content.search('.repo-leaderboard-meta .repo-leaderboard-meta-item .octicon-star'))
    project.fork_count  = meta_count(content.search('.repo-leaderboard-meta .repo-leaderboard-meta-item .octicon-git-branch'))

    projects << project
  end
  projects
end

#list_all_languages ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/git_trend/scraper.rb', line 36

def list_all_languages
  languages = []
  page = @agent.get(BASE_URL)
  page.search('div.select-menu-item a').each do |content|
    href = content.attributes['href'].value
    # objective-c++ =>
    language = href.match(/github.com\/trending\?l=(.+)/).to_a[1]
    languages << CGI.unescape(language) if language
  end
  languages
end