Module: Dumper
- Extended by:
- Alakazam
- Included in:
- Profile
- Defined in:
- lib/dumper/dumper.rb,
lib/dumper/logger.rb,
lib/dumper/profile.rb,
lib/dumper/version.rb,
lib/dumper/profiles.rb,
lib/dumper/profiles/fc2.rb,
lib/dumper/profiles/teca.rb,
lib/dumper/profiles/4chan.rb,
lib/dumper/profiles/booru.rb,
lib/dumper/profiles/fakku.rb,
lib/dumper/profiles/yande.rb,
lib/dumper/profiles/behoimi.rb,
lib/dumper/profiles/mangago.rb,
lib/dumper/profiles/gelbooru.rb,
lib/dumper/profiles/imagebam.rb,
lib/dumper/profiles/mangaeden.rb,
lib/dumper/profiles/mangahere.rb,
lib/dumper/profiles/deviantart.rb,
lib/dumper/profiles/multiplayer.rb,
lib/dumper/profiles/sankakucomplex.rb
Overview
– Copyright© 2013 Giovanni Capuano <[email protected]>
This file is part of Dumper.
Dumper is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
Dumper is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with Dumper. If not, see <www.gnu.org/licenses/>. ++
Defined Under Namespace
Modules: Profiles Classes: Logger, Profile
Constant Summary collapse
- USER_AGENT =
'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:36.0) Gecko/20100101 Firefox/36.0'
Class Method Summary collapse
- .get(path, url, options = {}) ⇒ Object
- .get_generic(url, path, xpath) ⇒ Object
- .list ⇒ Object
- .pool_size=(min, max = nil) ⇒ Object
- .shut_up! ⇒ Object (also: mute!)
- .verbose! ⇒ Object (also: unmute!)
- .verbose=(verbose) ⇒ Object
- .verbose? ⇒ Boolean
- .version ⇒ Object
Instance Method Summary collapse
- #method_missing(method, *args, &block) ⇒ Object
- #mute? ⇒ Boolean (also: #muted?)
- #pool_size ⇒ Object
- #verbose? ⇒ Boolean
Dynamic Method Handling
This class handles dynamic methods through the method_missing method
#method_missing(method, *args, &block) ⇒ Object
147 148 149 |
# File 'lib/dumper/dumper.rb', line 147 def method_missing(method, *args, &block) "'#{method.split('get_')[1]}' profile not found." end |
Class Method Details
.get(path, url, options = {}) ⇒ Object
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 |
# File 'lib/dumper/dumper.rb', line 72 def get(path, url, = {}) url = url.to_s errors = 0 begin if url.start_with? 'data:image/' filename = File.join path, [:filename] || rand(1000).to_s + '.' + url.split('data:image/')[1].split(?;)[0] filename.gsub! File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR url.gsub /data:image\/png;base64,/, '' if File.exists? filename notify_observers error: "File #{filename} already exists." else notify_observers status: "Downloading base64 image as #{filename}..." File.open(filename, 'wb') { |f| f.write Base64.decode64(url) } end else filename = File.join path, [:filename] || File.basename(url) filename.gsub! File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR if File.exists? filename notify_observers error: "File #{filename} already exists." else filename = File.join(path, rand(1000).to_s + '.jpg') unless filename[-4] == ?. || filename[-5] == ?. = { 'User-Agent' => [:user_agent] || USER_AGENT, 'Referer' => [:referer ] || url } url.prepend('http:') if url.start_with?('//') uri = URI(url) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = uri.scheme == 'https' http.verify_mode = OpenSSL::SSL::VERIFY_NONE http.start do |h| notify_observers status: "Downloading #{url} as #{filename}..." response = h.request Net::HTTP::Get.new(uri.request_uri, ) File.open(filename, 'wb') do |f| f.write response.body end end end end rescue Exception => e if errors <= 3 errors += 1 sleep 3 retry else notify_observers critical_error: "Error downloading #{url}.", critical_error_dump: e return false end end true end |
.get_generic(url, path, xpath) ⇒ Object
134 135 136 137 138 139 140 141 142 143 144 |
# File 'lib/dumper/dumper.rb', line 134 def get_generic(url, path, xpath) uri = nil Nokogiri::HTML(open(url)).xpath(xpath).each do |p| if p.to_s.start_with? ?/ uri = URI(url) if uri.nil? p = "#{uri.scheme}://#{uri.host}#{p}" end get path, p end end |
.list ⇒ Object
66 67 68 69 70 |
# File 'lib/dumper/dumper.rb', line 66 def list Dir.glob(File.('../profiles/*.rb', __FILE__)).sort { |a, b| b <=> a }.map do |f| f = File.basename(f).split(?.)[0] end end |
.pool_size=(min, max = nil) ⇒ Object
43 44 45 46 |
# File 'lib/dumper/dumper.rb', line 43 def pool_size=(min, max = nil) @min = min @max = max end |
.shut_up! ⇒ Object Also known as: mute!
52 53 54 |
# File 'lib/dumper/dumper.rb', line 52 def shut_up! @verbose == false end |
.verbose! ⇒ Object Also known as: unmute!
57 58 59 |
# File 'lib/dumper/dumper.rb', line 57 def verbose! @verbose == true end |
.verbose=(verbose) ⇒ Object
48 49 50 |
# File 'lib/dumper/dumper.rb', line 48 def verbose=(verbose) @verbose = verbose end |
.verbose? ⇒ Boolean
62 63 64 |
# File 'lib/dumper/dumper.rb', line 62 def verbose? @verbose == nil || @verbose == true end |
.version ⇒ Object
21 22 23 |
# File 'lib/dumper/version.rb', line 21 def self.version '0.8.5' end |
Instance Method Details
#mute? ⇒ Boolean Also known as: muted?
33 34 35 |
# File 'lib/dumper/dumper.rb', line 33 def verbose? @verbose == nil || @verbose == true end |
#pool_size ⇒ Object
23 24 25 26 27 28 |
# File 'lib/dumper/dumper.rb', line 23 def pool_size { min: @min || 4, max: @max } end |
#verbose? ⇒ Boolean
30 31 32 |
# File 'lib/dumper/dumper.rb', line 30 def verbose? @verbose == nil || @verbose == true end |