Class: HttpUtilities::Proxies::ProxySeeder

Inherits:
Object
  • Object
show all
Defined in:
lib/http_utilities/proxies/proxy_seeder.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ ProxySeeder

Returns a new instance of ProxySeeder.



6
7
8
9
10
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 6

def initialize
  self.protocols      =   ['http', 'socks5']
  self.proxy_types    =   ['public', 'shared', 'private']
  self.categories     =   ['L1', 'L2', 'L3', 'unspecified']
end

Instance Attribute Details

#categories ⇒ Object

Returns the value of attribute categories.



4
5
6
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 4

def categories
  @categories
end

#protocols ⇒ Object

Returns the value of attribute protocols.



4
5
6
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 4

def protocols
  @protocols
end

#proxy_types ⇒ Object

Returns the value of attribute proxy_types.



4
5
6
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 4

def proxy_types
  @proxy_types
end

Instance Method Details

#bulk_import_proxies(proxy_list, protocol, proxy_type, category) ⇒ Object 



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 28

def bulk_import_proxies(proxy_list, protocol, proxy_type, category)        
  columns     =   [:host, :port, :protocol, :proxy_type, :category]
  category    =   (category && !category.eql?('unspecified')) ? category : nil

  begin
    proxy_list.slice!(0..1000).each do |proxy|
      host              =   proxy[:host]
      port              =   proxy[:port]
      
      proxy             =   ::Proxy.where(host: host, port: port).first || ::Proxy.new
      proxy.host        =   host
      proxy.port        =   port
      proxy.protocol    =   protocol
      proxy.proxy_type  =   proxy_type
      proxy.category    =   category
      proxy.save
    end
    
  end while (proxy_list && proxy_list.any?)
end

#get_proxies_from_files(pattern) ⇒ Object 



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 72

def get_proxies_from_files(pattern)
  proxies = []
  file_paths = Dir.glob(pattern)

  file_paths.each do |file_path|
    proxy_rows = []
    File.open(file_path, 'r') {|f| proxy_rows = f.readlines("\n") }

    proxy_rows.each do |row|
      host, port  =   nil
      
      parts       =   row.include?(":") ? row.split(":") : nil
      
      if (parts && parts.any? && parts.size >= 2)
        host        =   parts.first
        port        =   parts.second.to_i
      end
      
      proxies << {:host => host, :port => port} if (host && port)
    end
  end

  return proxies
end

#get_seed_root ⇒ Object 



97
98
99
100
101
102
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 97

def get_seed_root
  rails_seed_root   =   defined?(Rails) ? "#{Rails.root}/db/seed_data/proxies/" : nil
  gem_seed_root     =   File.join(File.dirname(__FILE__), "../../generators/templates/seed_data/proxies/")
  
  return (rails_seed_root && File.exists?(rails_seed_root)) ? rails_seed_root : gem_seed_root
end

#import_proxies ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 16

def import_proxies
  proxy_data = parse_proxies

  proxy_data.each do |protocol, types|
    types.each do |type, categories|
      categories.each do |category, proxies|
        bulk_import_proxies(proxies, protocol, type, category)
      end
    end        
  end if (proxy_data && !proxy_data.empty?)
end

#parse_proxies ⇒ Object 



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 49

def parse_proxies
  proxies = {}

  self.protocols.each do |protocol|
    proxies[protocol] = {}

    self.proxy_types.each do |proxy_type|
      proxies[protocol][proxy_type] = {}
      proxies[protocol][proxy_type]['unspecified'] = []

      if (protocol.eql?("http"))
        self.categories.each do |category|
          proxies[protocol][proxy_type][category] = get_proxies_from_files("#{get_seed_root}#{protocol}/#{proxy_type}/#{category}/*.txt")
        end
      end
      
      proxies[protocol][proxy_type]['unspecified'] = proxies[protocol][proxy_type]['unspecified'] + get_proxies_from_files("#{get_seed_root}#{protocol}/#{proxy_type}/*.txt")
    end
  end

  return proxies
end

#seed ⇒ Object 



12
13
14
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 12

def seed
  import_proxies
end