Module: Codesake::Links::Api

Defined in:
lib/codesake/links/api.rb

Class Method Summary collapse

Class Method Details

.code(url, proxy) ⇒ Object 



19
20
21
22
 
# File 'lib/codesake/links/api.rb', line 19

def self.code(url, proxy)
  res = Links::Api.get(url, proxy)
  (res.nil?)? -1 : res.code
end

.follow(url, proxy) ⇒ Object 



61
62
63
64
 
# File 'lib/codesake/links/api.rb', line 61

def self.follow(url, proxy)
  l = Links::Api.links(url)
  l[0]
end

.get(url, proxy) ⇒ Object

include Links::Google



11
12
13
 
# File 'lib/codesake/links/api.rb', line 11

def self.get(url, proxy)
  return Links::Api.request({:url=>url, :proxy=>proxy, :method=>:get})
end

.head(url, proxy) ⇒ Object 



15
16
17
 
# File 'lib/codesake/links/api.rb', line 15

def self.head(url, proxy)
  return Links::Api.request({:url=>url, :proxy=>proxy, :method=>:head})
end

.human(code) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
 
# File 'lib/codesake/links/api.rb', line 66

def self.human(code)
  case code.to_i
  when 200
    return "Open"
  when 301
    return "Moved"
  when 404
    return "Non existent"
  when 401
    return "Closed"
  when 403
    return "Forbidden"
  when -1
    return "No answer"
  else
    return "Broken"
  end
end

.links(url, proxy) ⇒ Object 



24
25
26
27
28
29
30
31
32
 
# File 'lib/codesake/links/api.rb', line 24

def self.links(url, proxy)
  res = Links::Api.get(url, proxy)
  if res.nil?
    return []
  end
  doc = Nokogiri::HTML.parse(res.body)
  l = doc.css('a').map { |link| link['href'] }
  l
end

.robots(site) ⇒ Object

TESTING: SPIDERS, ROBOTS, AND CRAWLERS (OWASP-IG-001)



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/codesake/links/api.rb', line 35

def self.robots(site)

  site = 'http://'+site unless site.start_with? 'http://' or site.start_with? 'https://'
  

  allow_list = []
  disallow_list = []

  begin
    res=Net::HTTP.get_response(URI(site+'/robots.txt'))
    return {:status=>:KO, :allow_list=>[], :disallow_list=>[], :error=>"robots.txt response code was #{res.code}"} if (res.code != "200")


    res.body.split("\n").each do |line|

      disallow_list << line.split(":")[1].strip.chomp if (line.downcase.start_with?('disallow'))
      allow_list << line.split(":")[1].strip.chomp if (line.downcase.start_with?('allow'))

    end
  rescue Exception => e
    return {:status=>:KO, :allow_list=>[], :disallow_list=>[], :error=>e.message}
  end

  {:status=>:OK, :allow_list=>allow_list, :disallow_list=>disallow_list, :error=>""}
end