Class: FeedUsGrabber

Inherits:
Object
  • Object
show all
Defined in:
lib/app/models/feed_us_grabber.rb

Instance Method Summary collapse

Constructor Details

#initialize ⇒ FeedUsGrabber

Returns a new instance of FeedUsGrabber.



14
15
16
17
18
19
20
21
22
 
# File 'lib/app/models/feed_us_grabber.rb', line 14

def initialize
	@mintCacheIntervalUnit = CI_FOREVER
	@mstrCacheFolder = File.join(Rails.root.to_s,'tmp','CachedWebContent')
	@mstrCacheFileExt = ".cache"
	@mstrCacheGroup = ""
	@mstrStateFile = File.join(Rails.root.to_s,'tmp','FeedUsGrabberState')
	@mstrClientWhiteList = ""
	@mstrDebugOutput = ""
end

Instance Method Details

#addToDebugOutput(debugOutput, info) ⇒ Object 



363
364
365
366
 
# File 'lib/app/models/feed_us_grabber.rb', line 363

def addToDebugOutput(debugOutput, info)	    
    debugOutput << info + "<br />"
    puts " Debug Trace -> " + info
end

#autoCacheToFile ⇒ Object 



133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/app/models/feed_us_grabber.rb', line 133

def autoCacheToFile
	appendDebugOutput("Start autoCacheToFile.  bIsPostBack = #{@bIsPostBack}")		
	if @bIsPostBack
		return
	end

	appendDebugOutput("Cache command received = #{@mstrCacheCommand}")			
	if @mstrCacheCommand.nil? || @mstrCacheCommand == ''
		if (!self.cachedFileExists) or self.cacheFileIsExpired
			self.createCacheFile
		end		
	else
		if @mstrCacheCommand == "clear"
			if @mstrCacheGroup != ''
				self.clearCacheGroupFiles(@mstrCacheGroup)
			elsif @mstrCacheFolder
				self.clearCacheFolder(@mstrCacheFolder)
			end
		end
		if @mstrCacheCommand == 'clearall' || @mstrCacheCommand == CACHE_COMMAND_FORCE
			self.clearAllCachedFiles
		end
	end
end

#cachedFileExists ⇒ Object 



158
159
160
 
# File 'lib/app/models/feed_us_grabber.rb', line 158

def cachedFileExists()
	File.readable?(@mstrCachedFileName)
end

#cacheFileIsExpired ⇒ Object 



162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
 
# File 'lib/app/models/feed_us_grabber.rb', line 162

def cacheFileIsExpired()
	# Ruby 1.9.x Time.parse expects d/m/y
	strDate = File.mtime(@mstrCachedFileName).strftime('%d/%m/%Y %I:%M:%S %p')
	case @mintCacheIntervalUnit
	when CI_MINUTES then strUnit = 'n'
	when CI_HOURS then strUnit = 'h'
	when CI_DAYS then strUnit = 'd'
	when CI_FOREVER then return false
	end

	if self.datediff(strUnit,strDate,Time.now().strftime('%d/%m/%Y %I:%M:%S %p')) > @mintCacheIntervalLength
		return true
	else
		return false
	end
end

#canConnectToFeedUs ⇒ Object 



334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
 
# File 'lib/app/models/feed_us_grabber.rb', line 334

def canConnectToFeedUs()
	# Used for phone home check
	canConnect = false

	if @mstrCacheCommand == CACHE_COMMAND_FORCE
		canConnect = true
	else
		begin
			r = Net::HTTP.get_response(URI.parse(@mstrDynURL))
			if r.body.nil? == false && r.body != ERROR_RESPONSE
				canConnect = true
			end
		rescue
			logError("Unable to connect to Feed.Us.  Cache will not be cleared. URL #{@mstrDynURL}")
		end
	end

	return canConnect;
end

#clearAllCachedFiles ⇒ Object 



313
314
315
316
317
318
319
320
321
 
# File 'lib/app/models/feed_us_grabber.rb', line 313

def clearAllCachedFiles
	logMessage = "Clearing all caches at path = #{@mstrCacheFolder}" 		
	appendDebugOutput(logMessage)			
	logfile = File.open(File.join(Rails.root.to_s,'log','FeedUsGrabber.log'),'a');
	grabber_logger = FeedUsGrabberLogger.new(logfile)
	grabber_logger.info(logMessage)
	logfile.close
	self.clearCacheFolder(@mstrCacheFolder);
end

#clearCacheFolder(folder) ⇒ Object 



323
324
325
326
327
328
329
330
331
332
 
# File 'lib/app/models/feed_us_grabber.rb', line 323

def clearCacheFolder(folder)
	canConnect = canConnectToFeedUs()

	if canConnect == true
		appendDebugOutput("Clear cache folder can connect, removing cache from #{folder}")						
		FileUtils.rm_r Dir.glob("#{folder}/*")
	else
		logError("Unable to connect to Feed.Us. Cache will not be cleared. URL that was checked: #{@mstrDynURL}")
	end
end

#clearCacheGroupFiles(group) ⇒ Object 



296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
 
# File 'lib/app/models/feed_us_grabber.rb', line 296

def clearCacheGroupFiles(group)
	if group == '.' || group == '..'
		logfile = File.open(File.join(Rails.root.to_s,'log','FeedUsGrabber.log'),'a');
		grabber_logger = FeedUsGrabberLogger.new(logfile)
		grabber_logger.warn("someone requested to delete . OR .. ")
		logfile.close
		return
	end
	logMessage = "Clearing cache at group  #{group} at path #{File.join(@mstrCacheFolder,group)}"
 	appendDebugOutput(logMessage)
	logfile = File.open(File.join(Rails.root.to_s,'log','FeedUsGrabber.log'),'a');
	grabber_logger = FeedUsGrabberLogger.new(logfile)
	grabber_logger.info(logMessage)
	logfile.close
	self.clearCacheFolder(File.join(@mstrCacheFolder,group))
end

#createCacheFile ⇒ Object 



215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
 
# File 'lib/app/models/feed_us_grabber.rb', line 215

def createCacheFile()		
	sFile = @mstrCachedFileName
	# Fetch the url
	begin
		r = Net::HTTP.get_response(URI.parse(@mstrDynURL))
	rescue
		logfile = File.open(File.join(Rails.root.to_s,'log','FeedUsGrabber.log'),'a');
		grabber_logger = FeedUsGrabberLogger.new(logfile)

		grabber_logger.error("Unable to fetch URL #{@mstrDynURL}")
		logfile.close
		return;
	end

	if self.makeDirectory(File.join(@mstrCacheFolder,@mstrCacheGroup))
		File.open(sFile,'w') do |file|
			file.write(r.body)
		end
		return true
	end
	return false
end

#createStateFile ⇒ Object 



191
192
193
 
# File 'lib/app/models/feed_us_grabber.rb', line 191

def createStateFile()
	File.new(@mstrStateFile,'w')
end

#datediff(interval, datefrom, dateto, using_timestamps = false) ⇒ Object 



245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
 
# File 'lib/app/models/feed_us_grabber.rb', line 245

def datediff(interval, datefrom, dateto, using_timestamps = false)
	#$interval can be:
	#yyyy - Number of full years
	#q - Number of full quarters
	#m - Number of full months
	#y - Difference between day numbers
	#(eg 1st Jan 2004 is "1", the first day. 2nd Feb 2003 is "33". The datediff is "-32".)
	#d - Number of full days
	#w - Number of full weekdays
	#ww - Number of full weeks
	#h - Number of full hours
	#n - Number of full minutes
	#s - Number of full seconds (default)

	unless using_timestamps
		datefrom = Time.parse(datefrom).to_i
		dateto = Time.parse(dateto).to_i
	end

	difference = dateto - datefrom  # Difference in seconds
	if interval == 'd'
		datediff = (difference / 86400).floor
	elsif interval == 'h'
		datediff = (difference / 3600).floor
	elsif interval == 'n'
		datediff = (difference / 60).floor
	else
		datediff = difference
	end
	datediff
end

#getCacheCommand ⇒ Object 



48
49
50
 
# File 'lib/app/models/feed_us_grabber.rb', line 48

def getCacheCommand
	@mstrCacheCommand
end

#getCachedFileName ⇒ Object 



67
68
69
 
# File 'lib/app/models/feed_us_grabber.rb', line 67

def getCachedFileName
	@getCachedFileName
end

#getCacheFileQualifiers(strURLParams) ⇒ Object 



127
128
129
130
131
 
# File 'lib/app/models/feed_us_grabber.rb', line 127

def getCacheFileQualifiers(strURLParams)
	strResults = nil		
	strResults  = '_' + strURLParams.gsub(/[=&%\.\/:]+/,'_')
	strResults
end

#getCacheFolder ⇒ Object 



52
53
54
 
# File 'lib/app/models/feed_us_grabber.rb', line 52

def getCacheFolder
	@mstrCacheFolder
end

#getCacheGroup ⇒ Object 



64
65
66
 
# File 'lib/app/models/feed_us_grabber.rb', line 64

def getCacheGroup
	@mstrCacheGroup
end

#getCacheIntervalLength ⇒ Object 



75
76
77
 
# File 'lib/app/models/feed_us_grabber.rb', line 75

def getCacheIntervalLength
	@mintCacheIntervalLength
end

#getCacheIntervalUnit ⇒ Object 



36
37
38
 
# File 'lib/app/models/feed_us_grabber.rb', line 36

def getCacheIntervalUnit
	@mintCacheIntervalUnit
end

#getCacheNames(sURL) ⇒ Object 



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
 
# File 'lib/app/models/feed_us_grabber.rb', line 107

def getCacheNames(sURL)
	if (sURL =~ /\?/) > 0
		base = sURL.split('?')[0]
		params = sURL.split('?')[1]
	else
		base = sURL
		params = ''
	end
	#strip off the file extension
	base = base.chomp(File.extname(base))
	base.sub!('http://','')
	base.gsub!(/[\?:\/\.]/,'_')
	@mstrScriptBaseName = base
	@mstrCacheFileQualifiers = self.getCacheFileQualifiers(params)
	@mstrCachedFileName =File.join( @mstrCacheFolder,
	@mstrCacheGroup,
	@mstrScriptBaseName + "_" + @mstrCacheFileQualifiers + @mstrCacheFileExt)
	@mstrCachedFileNameShort = @mstrScriptBaseName + "_" + @mstrCacheFileQualifiers + @mstrCacheFileExt
end

#getClientWhiteList ⇒ Object 



90
91
92
93
 
# File 'lib/app/models/feed_us_grabber.rb', line 90

def getClientWhiteList
	self.readState
	@mstrClientWhiteList
end

#getDebugOutput ⇒ Object 



99
100
101
 
# File 'lib/app/models/feed_us_grabber.rb', line 99

def getDebugOutput
	@mstrDebugOutput
end

#getDynURL ⇒ Object 



103
104
105
 
# File 'lib/app/models/feed_us_grabber.rb', line 103

def getDynURL
	@mstrDynURL
end

#getIncludeFlag ⇒ Object 



60
61
62
 
# File 'lib/app/models/feed_us_grabber.rb', line 60

def getIncludeFlag
	@mblnInclude
end

#logError(contents) ⇒ Object 



354
355
356
357
358
359
360
361
 
# File 'lib/app/models/feed_us_grabber.rb', line 354

def logError(contents)
	logfile = File.open(File.join(Rails.root.to_s,'log','FeedUsGrabber.log'),'a');
	grabber_logger = FeedUsGrabberLogger.new(logfile)

	grabber_logger.error(contents)
	logfile.close
	appendDebugOutput(contents)
end

#makeDirectory(dir, mode = 0755) ⇒ Object 



238
239
240
241
242
243
 
# File 'lib/app/models/feed_us_grabber.rb', line 238

def makeDirectory(dir, mode = 0755)
	if File.directory?(dir) || FileUtils.mkdir_p(dir)
		return true
	end
	return false
end

#readState ⇒ Object 



187
188
189
 
# File 'lib/app/models/feed_us_grabber.rb', line 187

def readState()		
	readStateFile
end

#readStateFile ⇒ Object 



195
196
197
198
199
200
201
202
 
# File 'lib/app/models/feed_us_grabber.rb', line 195

def readStateFile()	
	if (self.stateFileExists)			
		File.open(@mstrStateFile, "r").each_line do |line|
			@mstrClientWhiteList = line
		end	
	end
	appendDebugOutput("Read from state file #{@mstrStateFile} value #{@mstrClientWhiteList}")
end

#renderCacheFromFile ⇒ Object 



277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
 
# File 'lib/app/models/feed_us_grabber.rb', line 277

def renderCacheFromFile
	sFile = @mstrCachedFileName
	data = ''
	begin
		File.open(sFile,'r') do |file|
			while temp_data = file.gets
				data = data + temp_data
			end
		end
		data
	rescue
		logfile = File.open(File.join(Rails.root.to_s,'log','FeedUsGrabber.log'),'a');
		grabber_logger = FeedUsGrabberLogger.new(logfile)
		grabber_logger.error("Unable to render/open #{@mstrCachedFileName}")
		logfile.close			
	end
end

#saveState ⇒ Object 



179
180
181
182
183
184
185
 
# File 'lib/app/models/feed_us_grabber.rb', line 179

def saveState
	appendDebugOutput("Does state file exist " + self.stateFileExists.to_s)
	if (!self.stateFileExists)
		self.createStateFile
	end
	self.writeStateFile
end

#setCacheCommand(strNewValue) ⇒ Object 



44
45
46
 
# File 'lib/app/models/feed_us_grabber.rb', line 44

def setCacheCommand(strNewValue)
	@mstrCacheCommand = strNewValue
end

#setCacheFileExt(strNewValue) ⇒ Object 



40
41
42
 
# File 'lib/app/models/feed_us_grabber.rb', line 40

def setCacheFileExt(strNewValue)
	@mstrCacheFileExt = strNewValue
end

#setCacheFolder(param) ⇒ Object 



28
29
30
 
# File 'lib/app/models/feed_us_grabber.rb', line 28

def setCacheFolder(param)
	@mstrCacheFolder = param
end

#setCacheGroup(param) ⇒ Object 



24
25
26
 
# File 'lib/app/models/feed_us_grabber.rb', line 24

def setCacheGroup(param)
	@mstrCacheGroup = param
end

#setCacheIntervalLength(param) ⇒ Object 



71
72
73
 
# File 'lib/app/models/feed_us_grabber.rb', line 71

def setCacheIntervalLength(param)
	@mintCacheIntervalLength = param
end

#setCacheIntervalUnit(param) ⇒ Object 



32
33
34
 
# File 'lib/app/models/feed_us_grabber.rb', line 32

def setCacheIntervalUnit(param)
	@mintCacheIntervalUnit = param
end

#setClientWhiteList(param) ⇒ Object 



84
85
86
87
88
 
# File 'lib/app/models/feed_us_grabber.rb', line 84

def setClientWhiteList(param)
	# Use file to store state across requests
	@mstrClientWhiteList = param
	self.saveState
end

#setDebugOutput(param) ⇒ Object 



95
96
97
 
# File 'lib/app/models/feed_us_grabber.rb', line 95

def setDebugOutput(param)
	@mstrDebugOutput = param
end

#setDynURL(param) ⇒ Object 



79
80
81
82
 
# File 'lib/app/models/feed_us_grabber.rb', line 79

def setDynURL(param)
	@mstrDynURL = param
	self.getCacheNames(param)
end

#setIncludeFlag(strNewValue) ⇒ Object 



56
57
58
 
# File 'lib/app/models/feed_us_grabber.rb', line 56

def setIncludeFlag(strNewValue)
	@mblnInclude = $strNewValue
end

#stateFileExists ⇒ Object 



211
212
213
 
# File 'lib/app/models/feed_us_grabber.rb', line 211

def stateFileExists()
	File.readable?(@mstrStateFile)
end

#writeStateFile ⇒ Object 



204
205
206
207
208
209
 
# File 'lib/app/models/feed_us_grabber.rb', line 204

def writeStateFile
	appendDebugOutput("Writing to state file #{@mstrStateFile} value #{@mstrClientWhiteList}")
	File.open(@mstrStateFile,'w') do |file|
		file.write(@mstrClientWhiteList)
	end
end