Class: Network

Inherits:
Object
  • Object
show all
Extended by:
Logging
Defined in:
lib/network.rb

Overview

This class does all networking in the program. It opens network-connections, retrieves information and files from a remote site.

Constant Summary collapse

TIMEOUT = 
10
MAX_FAIL = 
10
@@log = 
init_logger(STDOUT, $log_level)

Class Method Summary collapse

Methods included from Logging

init_logger, log_level=, log_target=

Methods included from File_Checking

#file_check, file_check

Class Method Details

.headers(url, opt = {}) ⇒ Object

currently unused



84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/network.rb', line 84

def self::headers(url, opt={})
	uri = URI.parse(url)
	req = Net::HTTP::Head.new(uri.path)
	headers = Hash.new
	http(uri.host, uri.port).start do |http|
		http.request(req) do |res|
			headers = res.to_hash
			@@log.debug('response to head-request ' << res.to_hash.to_s)
		end
	end
	return headers
end

.http(host, port) ⇒ Object

Returns a connection to ‘host’ via ‘port’, taking into account the

possibility of a system-wide proxy-setting.  For the time being, the
proxy must be defined in the environment variable 'http_proxy', like
this: http_proxy="http://proxy.url:port".
Example: http_proxy="http://localhost:8080"
Example: http_proxy="http://my.proxy.provider.com:3128"

If not defined in this way, no proxy is used and the connection will be
direct.



55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/network.rb', line 55

def self::http(host, port)
	hos = nil
	begin
		hos = RbConfig['host_os']
	rescue Exception => ex
		hos = RUBY_PLATFORM
	end

	connection = nil
	if(hos && /linux/ =~ hos )
		proxy = ENV['http_proxy']
		if(proxy)
			parray = proxy.split(':')
			if(parray.size == 3)
				http, phost, pport = parray
				if(phost.start_with?('//'))
					phost = phost[2..phost.size]
				end
				puts 'Using proxy at ' << phost << ', port ' << pport
				connection = Net::HTTP.new(host, nil, phost, pport)
			end
		end
	end
	puts 'host: ' << host << ', port: ' << port << ', direct connection, no useable proxy-setting found' if !connection
	connection ||= Net::HTTP.new(host,port)
	return connection
end

.http_to_file(filename, url, opts = {}) ⇒ Object

Download url, save to filename.

Copied&Pasted from https://stackoverflow.com/questions/2263540/how-do-i-download-a-binary-file-over-http/6964173#6964173



101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
 
# File 'lib/network.rb', line 101

def self::http_to_file( filename, url, opts={} )
	opt = {
		:init_pause => 0.1,     # start by waiting this long each time
		# it's deliberately long so we can see
		# what a full buffer looks like
		:learn_period => 0.3,   # keep the initial pause for at least this many seconds
		:drop => 1.5,           # fast reducing factor to find roughly optimized pause time
		:adjust => 1.05,        # during the normal period, adjust up or down by this factor
		:continue => false,      # by default do not retry failed downloads
		:previous_size => 0      # by default consider this a new file to download.
	}.merge(opts)

	@@log.debug('http_to_file, merged opts are ' << opt.inspect)
	pause = opt[:init_pause]
	learn = 1 + (opt[:learn_period]/pause).to_i
	drop_period = true
	delta = 0
	max_delta = 0
	continue = opt[:continue]
	previous_size = opt[:previous_size]

	length = 0
	@@log.level = $log_level 
	@@log.debug('writing to file (a)' << filename)
	File.open(filename.shellescape,'a') do |f|
		f.flock(File::LOCK_EX)
		# exception handling (interrupt)
		begin
			uri = URI.parse(url)
			@@log.debug('uri is ' << uri.to_s)
			@@log.debug('uri.path is ' << uri.path)

			# headers = headers(uri)
			# @@log.debug('headers is ' << headers.inspect)
		
			req = Net::HTTP::Get.new(uri.path)
			@@log.debug('request is ' << req.inspect)
			# see: https://stackoverflow.com/questions/3303029/http-range-header
			req['Range'] = "bytes=#{f.size}-";
			@@log.debug('range will be ' << req['Range'].to_s)
			# begin loop, in case that continue == true
			count = 0
			# loop
			num_failures = 0
			begin
				# exception handling (network errors)
				begin 
					puts "Attempt #%i) " %(count += 1) if continue
					# -------- begin download
					http(uri.host, uri.port).start do |http|
						bi = nil
						dp = nil
						http.request(req) do |res|
							@@log.debug 'response is ' << res.inspect
							case res
							when Net::HTTPSuccess
								# reset error-count.
								num_failures = 0
								# if(headers.has_key?('content-Length'))
								if(res.key?('content-Length'))
									@@log.debug('content-Length is ' << (res["Content-Length"] ? res["Content-Length"] : 'N I L'))
									@@log.debug('file-size is: ' << (f ? f.size.to_s : 'N I L'))
									length = res["Content-Length"].to_i
									if(length == 0)
										puts ("No new content, file may be complete, already. Aborting")
										exit true
									elsif( f.size == 0 && previous_size == 0)
										@@log.debug('updating history ---->')
										$HISTORY.update(url, :size => length)
										@@log.debug('<----- did it')
										previous_size = length;
									end
								else
									@@log.warn('response without content-length')
									previous_size = f.size
								end
							else
								@@log.warn('received http response ' << res.class.name << ', code ' << res.code.to_s)
								@@log.error('aborting')
								exit false
							end
							# puts byte_units(length)
							puts "Fetching %s" %byte_units(length)
							#bi = BusyIndicator.new(true, 20) {sleep 0.4; byte_units(f.size, 3) << ' bytes of ' << byte_units($HISTORY[url].size, 3)  }
							dp = DownloadProgress.new(f, $HISTORY[url].size)
							dp.start
							#bi = BusyIndicator.new(true, 20) {sleep 0.4; dp.progress_bar  }
							res.read_body do |seg|
								f << seg
								$HISTORY.update(url, :position => f.size)

								if delta > max_delta then max_delta = delta end
								if learn <= 0 then
									learn -= 1
								elsif delta == max_delta then
									if drop_period then
										pause /= opt[:drop]
									else
										pause /= opt[:adjust]
									end
								elsif delta < max_delta then
									drop_period = false
									pause *= opt[:adjust]
								end
								sleep(pause)
							end
						end
						if (length && f.size >= length)
							@@log.info('removing download from history')
							$HISTORY.delete(url)
							bi.stop('Okay') if bi
							dp.stop
						else
							bi.stop("Transfer interrupted prematurely at %s of %s." %[byte_units(f.size, 3), byte_units(length, 3)])
							dp.stop
							puts "Will try to continue the download" if continue
						end

					end
					# ---------- end download
				rescue Exception => ex
					if ex.respond_to?(:signo)						
						raise ex
					end
					num_failures += 1
					@@log.error('An error occured while processing the response: ' << ex.message)
					if(num_failures < MAX_FAIL)
						@@log.warn('I will try to continue the download where it was interrupted')
					else
						@@log.error("#{MAX_FAIL} attempts to download have failed. Aborting for now. You should try again, later.")
					end
				end
			end while( (f.size < length) && continue)
			# end loop, if continue == true
		rescue Interrupt 
			puts "\n\tProgram interrupted\n"
			exit 0
		end

	end # file.open
end