Class: HttpUtilities::Proxies::ProxySeeder

Inherits:
Object
  • Object
show all
Defined in:
lib/http_utilities/proxies/proxy_seeder.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ ProxySeeder

Returns a new instance of ProxySeeder.



7
8
9
10
11
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 7

def initialize
  self.protocols      =   ['http', 'socks5']
  self.proxy_types    =   ['public', 'shared', 'private']
  self.categories     =   ['L1', 'L2', 'L3', 'unspecified']
end

Instance Attribute Details

#categories ⇒ Object

Returns the value of attribute categories.



5
6
7
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 5

def categories
  @categories
end

#protocols ⇒ Object

Returns the value of attribute protocols.



5
6
7
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 5

def protocols
  @protocols
end

#proxy_types ⇒ Object

Returns the value of attribute proxy_types.



5
6
7
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 5

def proxy_types
  @proxy_types
end

Instance Method Details

#bulk_import_proxies(proxy_list, protocol, proxy_type, category) ⇒ Object 



29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 29

def bulk_import_proxies(proxy_list, protocol, proxy_type, category)        
  columns = [:host, :port, :protocol, :proxy_type, :category]
  category = (category && !category.eql?('unspecified')) ? category : nil

  begin
    values = []

    proxy_list.slice!(0..1000).each do |proxy|
      host = proxy[:host]
      port = proxy[:port]
      value_arr = [host, port, protocol, proxy_type, category]
      values << value_arr
    end

    ::Proxy.import(columns, values, :on_duplicate_key_update => [:proxy_type], :validate => false) if (values && values.any?)
  end while (proxy_list && proxy_list.any?)
end

#get_proxies_from_files(pattern) ⇒ Object 



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 70

def get_proxies_from_files(pattern)
  proxies = []
  file_paths = Dir.glob(pattern)

  file_paths.each do |file_path|
    proxy_rows = []
    File.open(file_path, 'r') {|f| proxy_rows = f.readlines("\n") }

    proxy_rows.each do |row|
      host, port  =   nil
      
      parts       =   row.include?(":") ? row.split(":") : nil
      
      if (parts && parts.any? && parts.size >= 2)
        host        =   parts.first
        port        =   parts.second.to_i
      end
      
      proxies << {:host => host, :port => port} if (host && port)
    end
  end

  return proxies
end

#get_seed_root ⇒ Object 



95
96
97
98
99
100
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 95

def get_seed_root
  rails_seed_root   =   defined?(Rails) ? "#{Rails.root}/db/seed_data/proxies/" : nil
  gem_seed_root     =   File.join(File.dirname(__FILE__), "../../generators/templates/seed_data/proxies/")
  
  return (rails_seed_root && File.exists?(rails_seed_root)) ? rails_seed_root : gem_seed_root
end

#import_proxies ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 17

def import_proxies
  proxy_data = parse_proxies

  proxy_data.each do |protocol, types|
    types.each do |type, categories|
      categories.each do |category, proxies|
        bulk_import_proxies(proxies, protocol, type, category)
      end
    end        
  end if (proxy_data && !proxy_data.empty?)
end

#parse_proxies ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 47

def parse_proxies
  proxies = {}

  self.protocols.each do |protocol|
    proxies[protocol] = {}

    self.proxy_types.each do |proxy_type|
      proxies[protocol][proxy_type] = {}
      proxies[protocol][proxy_type]['unspecified'] = []

      if (protocol.eql?("http"))
        self.categories.each do |category|
          proxies[protocol][proxy_type][category] = get_proxies_from_files("#{get_seed_root}#{protocol}/#{proxy_type}/#{category}/*.txt")
        end
      end
      
      proxies[protocol][proxy_type]['unspecified'] = proxies[protocol][proxy_type]['unspecified'] + get_proxies_from_files("#{get_seed_root}#{protocol}/#{proxy_type}/*.txt")
    end
  end

  return proxies
end

#seed ⇒ Object 



13
14
15
 
# File 'lib/http_utilities/proxies/proxy_seeder.rb', line 13

def seed
  import_proxies
end