Class: Apollo::BaseProgram

Inherits:
Object
  • Object
show all
Defined in:
lib/apollo_crawler/program/base_program.rb

Direct Known Subclasses

ConsoleProgram, CrawlerProgram, PlatformProgram

Constant Summary collapse

CONFIG_DIR = 
File.join(Apollo::BASE_DIR, "config")
DEFAULT_OPTIONS = 
{
	:env => Apollo::ENV,
	:verbose => false
}

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ BaseProgram

Returns a new instance of BaseProgram.



41
42
43
44
45
46
47
48
 
# File 'lib/apollo_crawler/program/base_program.rb', line 41

def initialize
	self.config = {}
	self.options = DEFAULT_OPTIONS
	self.optparser = nil

	self.amqp = nil
	self.mongo = nil
end

Instance Attribute Details

#amqp ⇒ Object

Returns the value of attribute amqp.



38
39
40
 
# File 'lib/apollo_crawler/program/base_program.rb', line 38

def amqp
  @amqp
end

#config ⇒ Object

Returns the value of attribute config.



34
35
36
 
# File 'lib/apollo_crawler/program/base_program.rb', line 34

def config
  @config
end

#mongo ⇒ Object

Returns the value of attribute mongo.



39
40
41
 
# File 'lib/apollo_crawler/program/base_program.rb', line 39

def mongo
  @mongo
end

#options ⇒ Object

Returns the value of attribute options.



35
36
37
 
# File 'lib/apollo_crawler/program/base_program.rb', line 35

def options
  @options
end

#optparser ⇒ Object

Returns the value of attribute optparser.



36
37
38
 
# File 'lib/apollo_crawler/program/base_program.rb', line 36

def optparser
  @optparser
end

Class Method Details

.get_config_path(config) ⇒ Object 



50
51
52
 
# File 'lib/apollo_crawler/program/base_program.rb', line 50

def self.get_config_path(config)
	return File.join(CONFIG_DIR, "#{config}.yml")
end

.require_files(files = []) ⇒ Object 



86
87
88
89
90
 
# File 'lib/apollo_crawler/program/base_program.rb', line 86

def self.require_files(files = [])
	Dir.glob(files).each do |file|
		require file
	end
end

Instance Method Details

#init_amqp ⇒ Object 



118
119
120
121
122
123
124
125
 
# File 'lib/apollo_crawler/program/base_program.rb', line 118

def init_amqp()
	conn_opts = self.config["amqp"]
	if(conn_opts)
		self.amqp = Apollo::Helper::Amqp::connect(conn_opts, self.options)
	end

	return self.amqp
end

#init_mongo ⇒ Object 



127
128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/apollo_crawler/program/base_program.rb', line 127

def init_mongo()
	conn_opts = self.config["mongo"]
	if(conn_opts)
		self.mongo = Apollo::Helper::Mongo::connect(conn_opts, self.options)

		# Init Mongoid
		path = File.join(Apollo::BASE_DIR, "config/mongoid.yml")
		Mongoid.load!(path, @options[:env])
	end

	return self.mongo
end

#init_options ⇒ Object 



114
115
116
 
# File 'lib/apollo_crawler/program/base_program.rb', line 114

def init_options()
	return nil
end

#init_program(args) ⇒ Object

Init program



168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/apollo_crawler/program/base_program.rb', line 168

def init_program(args)
	res = nil
	res = init_options()

	begin
		res = parse_options(args)
	rescue Exception => e
		puts "Unable to parse options"
		return res unless res.nil?
	end

	res = process_options(args)
	return res unless res.nil?

	confs = load_configs(BaseProgram::CONFIG_DIR, @options)
	puts "Loaded configs => #{confs.inspect}" if @options[:verbose]

	# Init Mongo Connection
	init_mongo()

	# Init AMQP
	init_amqp()

	# Init Seed data
	init_seeds(@options)

	return nil
end

#init_seeds(opts = {}) ⇒ Object 



163
164
165
 
# File 'lib/apollo_crawler/program/base_program.rb', line 163

def init_seeds(opts={})
	init_seeds_crawlers(opts)
end

#init_seeds_crawlers(opts = {}) ⇒ Object 



140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
 
# File 'lib/apollo_crawler/program/base_program.rb', line 140

def init_seeds_crawlers(opts={})
	objs = Apollo::Crawler::BaseCrawler.subclasses
	objs.each do |o|
		crawler = Apollo::Model::Crawler.new
		i = o.new
		crawler.name = i.name
		crawler.class_name = o.to_s
		
		res = Apollo::Model::Crawler.where(class_name: crawler.class_name)
		# puts "RES: '#{res.inspect}'"
		if(res.nil? || res.count < 1)
			crawler.save
			if(opts[:verbose])
				puts "Adding new crawler - '#{crawler.inspect}'"
			end
		else
			if(opts[:verbose])
				puts "Using crawler - '#{res[0].inspect}'"
			end
		end
	end
end

#load_config(config_name, options = DEFAULT_OPTIONS) ⇒ Object 



81
82
83
84
 
# File 'lib/apollo_crawler/program/base_program.rb', line 81

def load_config(config_name, options = DEFAULT_OPTIONS)
	path = BaseProgram.get_config_path(config_name)
	return load_config_file(path, options)
end

#load_config_file(config_file, options = DEFAULT_OPTIONS) ⇒ Object 



63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/apollo_crawler/program/base_program.rb', line 63

def load_config_file(config_file, options = DEFAULT_OPTIONS)
	return nil unless File.exists?(config_file)

	config_name = File.basename(config_file.downcase, ".yml")

	res = YAML.load(File.read(config_file))
	return nil unless res

	res = {config_name => res[options[:env]]}
	self.config.merge! res
	
	if(options[:verbose])
		puts "Loaded config '#{config_file}' => #{res[config_name]}"
	end

	return res[config_name]
end

#load_configs(dir = CONFIG_DIR, options = DEFAULT_OPTIONS) ⇒ Object 



54
55
56
57
58
59
60
61
 
# File 'lib/apollo_crawler/program/base_program.rb', line 54

def load_configs(dir = CONFIG_DIR, options = DEFAULT_OPTIONS)
	Dir[File.join(dir, "**/*.yml")].each do |c|
		puts "Loading config #{c}" if options[:verbose]
		self.load_config_file(c, options)
	end

	return self.config
end

#parse_options(args = ARGV) ⇒ Object

Parse the options passed to command-line



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/apollo_crawler/program/base_program.rb', line 93

def parse_options(args = ARGV)
	res = []

	# Parse the command-line. Remember there are two forms
	# of the parse method. The 'parse' method simply parses
	# ARGV, while the 'parse!' method parses ARGV and removes
	# any options found there, as well as any parameters for
	# the options. What's left is the list of files to resize.
	begin
		optparser.parse!(args)
	rescue Exception => e
		return nil
	end

	return res
end

#process_options(args) ⇒ Object 



110
111
112
 
# File 'lib/apollo_crawler/program/base_program.rb', line 110

def process_options(args)
	return nil
end

#request_exit(code = 0) ⇒ Object 



205
206
207
208
209
210
211
212
213
214
 
# File 'lib/apollo_crawler/program/base_program.rb', line 205

def request_exit(code = 0)
	puts "Exiting app"
	begin
		exit(code)
	rescue SystemExit => e
		# puts "rescued a SystemExit exception, reason: '#{e.to_s}'"
	end

	return code
end

#run(args = ARGV) ⇒ Object

Run Program



198
199
200
201
202
203
 
# File 'lib/apollo_crawler/program/base_program.rb', line 198

def run(args = ARGV)
	res = init_program(args)
	return res unless res.nil?

	puts "Running environment '#{@options[:env]}'" if @options[:verbose]
end