Class: Wmap::SiteTracker::WpTracker
- Inherits:
-
Wmap::SiteTracker
- Object
- Wmap::SiteTracker
- Wmap::SiteTracker::WpTracker
- Includes:
- Utils, Utils::WpDetect
- Defined in:
- lib/wmap/site_tracker/wp_tracker.rb
Constant Summary
Constants included from Utils::UrlMagic
Utils::UrlMagic::Max_http_timeout, Utils::UrlMagic::User_agent
Constants included from Utils::DomainRoot
Utils::DomainRoot::File_ccsld, Utils::DomainRoot::File_cctld, Utils::DomainRoot::File_gtld, Utils::DomainRoot::File_tld
Instance Attribute Summary collapse
-
#data_dir ⇒ Object
include Singleton.
-
#http_timeout ⇒ Object
include Singleton.
-
#known_wp_sites ⇒ Object
readonly
Returns the value of attribute known_wp_sites.
-
#max_parallel ⇒ Object
include Singleton.
-
#sites_wp ⇒ Object
include Singleton.
-
#verbose ⇒ Object
include Singleton.
Attributes inherited from Wmap::SiteTracker
Instance Method Summary collapse
-
#add(url, use_cache = true) ⇒ Object
Add wordpress entry to the cache one at a time.
-
#bulk_add(list, num = @max_parallel, use_cache = true) ⇒ Object
(also: #adds)
Method to load wp sites in parallel.
-
#initialize(params = {}) ⇒ WpTracker
constructor
WordPress checker instance default variables.
-
#load_from_file(file = @sites_wp, lc = true) ⇒ Object
‘setter’ to load the known wordpress sites into an instance variable.
-
#refresh(target, use_cache = false) ⇒ Object
Refresh one site entry then update the instance variable (cache).
-
#refreshs(num = @max_parallel, use_cache = false) ⇒ Object
Refresh wordpress site entries within the sitetracker list.
-
#save_to_file!(file_wps = @sites_wp, wps = @known_wp_sites) ⇒ Object
(also: #save!)
Save the current hash table into a file.
Methods included from Utils::WpDetect
#is_wp?, #wp_css?, #wp_gen?, #wp_load_styles?, #wp_login?, #wp_meta?, #wp_readme?, #wp_rpc?, #wp_ver, #wp_ver_generator, #wp_ver_load_styles, #wp_ver_login, #wp_ver_meta, #wp_ver_readme
Methods included from Utils
#cidr_2_ips, #file_2_hash, #file_2_list, #get_nameserver, #get_nameservers, #host_2_ip, #host_2_ips, #is_cidr?, #is_fqdn?, #is_ip?, #list_2_file, #reverse_dns_lookup, #sort_ips, #valid_dns_record?, #zone_transferable?
Methods included from Utils::Logger
Methods included from Utils::UrlMagic
#create_absolute_url_from_base, #create_absolute_url_from_context, #host_2_url, #is_site?, #is_ssl?, #is_url?, #landing_location, #make_absolute, #normalize_url, #open_page, #redirect_location, #response_code, #response_headers, #url_2_host, #url_2_path, #url_2_port, #url_2_site, #urls_on_same_domain?
Methods included from Utils::DomainRoot
#get_domain_root, #get_domain_root_by_ccsld, #get_domain_root_by_cctld, #get_domain_root_by_tlds, #get_sub_domain, #is_domain_root?, #print_ccsld, #print_cctld, #print_gtld
Methods inherited from Wmap::SiteTracker
#bulk_delete, #bulk_refresh, #count, #delete, #file_add, #file_delete, #file_refresh, #get_ext_sites, #get_int_sites, #get_ip_sites, #get_prim_uniq_sites, #get_redirection_url, #get_redirection_urls, #get_ssl_sites, #get_uniq_sites, #load_site_stores_from_file, #print_all_sites, #print_ext_sites, #print_int_sites, #print_ip_sites, #print_site, #print_ssl_sites, #print_uniq_sites, #refresh_all, #refresh_ip_sites, #refresh_uniq_sites, #resolve_ip_sites, #save_sites_to_file!, #save_uniq_sites, #save_uniq_sites_xml, #search, #site_check, #site_ip_known?, #site_known?
Constructor Details
#initialize(params = {}) ⇒ WpTracker
WordPress checker instance default variables
24 25 26 27 28 29 30 31 32 33 34 35 |
# File 'lib/wmap/site_tracker/wp_tracker.rb', line 24 def initialize (params = {}) @verbose=params.fetch(:verbose, false) @data_dir=params.fetch(:data_dir, File.dirname(__FILE__)+'/../../../data/') Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir) @sites_wp=params.fetch(:sites_wp, @data_dir+"wp_sites") @http_timeout=params.fetch(:http_timeout, 5000) @max_parallel=params.fetch(:max_parallel, 40) Dir.mkdir(@data_dir) unless Dir.exist?(@data_dir) @log_file=@data_dir + "wp_checker.log" File.write(@sites_wp, "") unless File.exist?(@sites_wp) load_from_file(@sites_wp) end |
Instance Attribute Details
#data_dir ⇒ Object
include Singleton
20 21 22 |
# File 'lib/wmap/site_tracker/wp_tracker.rb', line 20 def data_dir @data_dir end |
#http_timeout ⇒ Object
include Singleton
20 21 22 |
# File 'lib/wmap/site_tracker/wp_tracker.rb', line 20 def http_timeout @http_timeout end |
#known_wp_sites ⇒ Object (readonly)
Returns the value of attribute known_wp_sites.
21 22 23 |
# File 'lib/wmap/site_tracker/wp_tracker.rb', line 21 def known_wp_sites @known_wp_sites end |
#max_parallel ⇒ Object
include Singleton
20 21 22 |
# File 'lib/wmap/site_tracker/wp_tracker.rb', line 20 def max_parallel @max_parallel end |
#sites_wp ⇒ Object
include Singleton
20 21 22 |
# File 'lib/wmap/site_tracker/wp_tracker.rb', line 20 def sites_wp @sites_wp end |
#verbose ⇒ Object
include Singleton
20 21 22 |
# File 'lib/wmap/site_tracker/wp_tracker.rb', line 20 def verbose @verbose end |
Instance Method Details
#add(url, use_cache = true) ⇒ Object
Add wordpress entry to the cache one at a time
86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 |
# File 'lib/wmap/site_tracker/wp_tracker.rb', line 86 def add(url, use_cache=true) puts "Add entry to the local cache table: #{url}" if @verbose site=url_2_site(url) if use_cache && @known_wp_sites.key?(site) puts "Site is already exist. Skipping: #{site}" else record=Hash.new redirection = landing_location(site) if not [nil, ''].include?(redirection) if is_wp?(redirection) version = wp_ver(redirection) record['site'] = site record['version'] = version record['redirection'] = redirection @known_wp_sites[site]=record puts "Entry added: #{record}" end else if is_wp?(site) version = wp_ver(site) record['version'] = version record['redirection'] = redirection @known_wp_sites[site]=record puts "Entry added: #{record}" end end end return record rescue => ee puts "Exception on method #{__method__}: #{ee}: #{url}" if @verbose end |
#bulk_add(list, num = @max_parallel, use_cache = true) ⇒ Object Also known as: adds
Method to load wp sites in parallel
119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 |
# File 'lib/wmap/site_tracker/wp_tracker.rb', line 119 def bulk_add(list,num=@max_parallel,use_cache=true) puts "Add entries to the local wp_site store from list:\n #{list}" results=Hash.new list = list - [nil,""] if list.size > 0 puts "Start parallel adding on the sites:\n #{list}" Parallel.map(list, :in_processes => num) { |target| add(target,use_cache) }.each do |process| if process.nil? next elsif process.empty? next #do nothing else results[process['site']]=Hash.new results[process['site']]=process end end @known_wp_sites.merge!(results) else puts "Error: no entry is added. Please check your list and try again." end puts "Done adding site entries." if results.size>0 puts "New entries added: #{results}" else puts "No new entry added. " end return results rescue => ee puts "Exception on method #{__method__}: #{ee}" if @verbose end |
#load_from_file(file = @sites_wp, lc = true) ⇒ Object
‘setter’ to load the known wordpress sites into an instance variable
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 |
# File 'lib/wmap/site_tracker/wp_tracker.rb', line 38 def load_from_file (file=@sites_wp, lc=true) puts "Loading trusted file: #{file}" if @verbose @known_wp_sites=Hash.new f_wp_sites=File.open(file, 'r') f_wp_sites.each_line do |line| puts "Processing line: #{line}" if @verbose line=line.chomp.strip next if line.nil? next if line.empty? next if line =~ /^\s*#/ line=line.downcase if lc==true entry=line.split(',') site = entry[0].strip() next if site.nil? if @known_wp_sites.key?(site) next else @known_wp_sites[site] = Hash.new @known_wp_sites[site]['site'] = site @known_wp_sites[site]['version'] = entry[1].strip() @known_wp_sites[site]['redirection'] = entry[2].strip() end end f_wp_sites.close return @known_wp_sites rescue => ee puts "Exception on method #{__method__}: #{ee}" if @verbose return Hash.new end |
#refresh(target, use_cache = false) ⇒ Object
Refresh one site entry then update the instance variable (cache)
154 155 156 |
# File 'lib/wmap/site_tracker/wp_tracker.rb', line 154 def refresh (target,use_cache=false) return add(target,use_cache) end |
#refreshs(num = @max_parallel, use_cache = false) ⇒ Object
Refresh wordpress site entries within the sitetracker list
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 |
# File 'lib/wmap/site_tracker/wp_tracker.rb', line 159 def refreshs (num=@max_parallel,use_cache=false) puts "Add entries to the local cache table from site tracker: " if @verbose results=Hash.new wps=@known_wp_sites.keys if wps.size > 0 Parallel.map(wps, :in_processes => num) { |target| refresh(target,use_cache) }.each do |process| if process.nil? next elsif process.empty? #do nothing else site = process['site'] results[site] = process end end @known_wp_sites.merge!(results) puts "Done loading entries." return results else puts "Error: no entry is loaded. Please check your list and try again." end wps=nil return results #rescue => ee # puts "Exception on method #{__method__}: #{ee}" if @verbose # return Hash.new end |
#save_to_file!(file_wps = @sites_wp, wps = @known_wp_sites) ⇒ Object Also known as: save!
Save the current hash table into a file
69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
# File 'lib/wmap/site_tracker/wp_tracker.rb', line 69 def save_to_file!(file_wps=@sites_wp, wps=@known_wp_sites) puts "Saving the current wordpress site table from memory to file: #{file_wps} ..." if @verbose =Time.now f=File.open(file_wps, 'w') f.write "# Local wps file created by class #{self.class} method #{__method__} at: #{}\n" f.write "# WP Site URL, WP Version, Redirection \n" (wps.keys - [nil,'']).sort.map do |key| f.write "#{key}, #{wps[key]['version']}, #{wps[key]['redirection']}\n" end f.close puts "WordPress site cache table is successfully saved: #{file_wps}" rescue => ee puts "Exception on method #{__method__}: #{ee}" if @verbose end |