Module: Meiriyigua
- Defined in:
- lib/meiriyigua.rb,
lib/meiriyigua/config.rb,
lib/meiriyigua/models.rb,
lib/meiriyigua/version.rb,
lib/meiriyigua/list_crawl.rb,
lib/meiriyigua/baidu_crawl.rb,
lib/meiriyigua/post_client.rb,
lib/meiriyigua/crawl_client.rb,
lib/meiriyigua/detail_crawl.rb,
lib/meiriyigua/post_client_two.rb
Defined Under Namespace
Modules: Models
Classes: BaiduCrawl, Config, CrawlClient, DetailCrawl, ListCrawl, PostClient, PostClientTwo
Constant Summary
collapse
- VERSION =
"0.0.5"
Class Method Summary
collapse
Class Method Details
.__run ⇒ Object
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
# File 'lib/meiriyigua.rb', line 23
def __run
Thread.abort_on_exception = true
while true
begin
((Meiriyigua::Models::UrlRecord.all(baidu_at: nil) | Meiriyigua::Models::UrlRecord.all(publish_at: nil)) & Meiriyigua::Models::UrlRecord.all(:detail_at.lt => (Date.today - 7))).each do |url_record|
url_record.page_record.destroy
url_record.destroy
end
Meiriyigua::Models::PageRecord.all(url_record: {:publish_at.not => nil}).destroy!
threads = []
threads << Thread.new do
list_crawl = ListCrawl.new
list_crawl.run
detail_crawl = DetailCrawl.new(list_crawl.detail_urls)
detail_crawl.run
sleep(Meiriyigua::Config.update_time.to_i)
end
threads << Thread.new do
baidu_crawl = BaiduCrawl.new
baidu_crawl.run
sleep(Meiriyigua::Config.update_time.to_i)
end
threads << Thread.new do
if @type == 1
post_client = PostClient.new
post_client.run
else
post_client = PostClientTwo.new
post_client.run
end
sleep(Meiriyigua::Config.post_time.to_i)
end
threads.each { |thr| thr.join }
rescue
puts "出错了 #{$!.message}\n#{$!.backtrace.join("\n")}"
threads.each { |thr| thr.exit rescue nil }
end
sleep 60*10
end
end
|
.run ⇒ Object
12
13
14
15
|
# File 'lib/meiriyigua.rb', line 12
def run
@type = 1
__run
end
|
.run_two ⇒ Object
17
18
19
20
|
# File 'lib/meiriyigua.rb', line 17
def run_two
@type = 2
__run
end
|