Module: Javlibrary

Defined in:
lib/javlibrary.rb,
lib/javlibrary/init.rb,
lib/javlibrary/name.rb,
lib/javlibrary/video.rb,
lib/javlibrary/version.rb,
lib/javlibrary/database.rb,
lib/javlibrary/info_hash.rb,
lib/javlibrary/downloader.rb

Constant Summary collapse

VERSION = 
"0.1.7"

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.actor_hash ⇒ Object 



113
114
115
116
117
118
119
120
121
122
 
# File 'lib/javlibrary.rb', line 113

def actor_hash
    client = Javlibrary.client
    actor_hash = Hash.new
    client.query("SELECT * FROM actor").each do |item|
        actor_hash["#{item['actor_name']}"] = item['actor_id']
    end
    client.close

    actor_hash
end

.author_page_num(nokogiri_doc) ⇒ Object 



163
164
165
166
167
168
169
 
# File 'lib/javlibrary.rb', line 163

def Javlibrary.author_page_num(nokogiri_doc)
    last_page = 1
    nokogiri_doc.search('//div[@class="page_selector"]/a[@class="page last"]').each do |row|
        last_page = row['href'].split("=")[-1].to_i
    end
    last_page
end

.client ⇒ Object 



10
11
12
13
14
15
 
# File 'lib/javlibrary.rb', line 10

def client
    client = Mysql2::Client.new(:host => "127.0.0.1",
                                :username => "root",
                                :password => "XuHefeng",
                                :database => "javlibrary_new")
end

.download_all_video ⇒ Object 



79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/javlibrary.rb', line 79

def download_all_video
    client = Javlibrary.client
    result = client.query("SELECT video_num, video_label FROM label WHERE video_download=0")
    client.close
    
    video_array = Array.new
    result.each do |item|
        video_array << item
    end

    video_array = video_array.each_slice(5000).to_a
    actor_hash = Javlibrary::actor_hash
    genre_hash = Javlibrary::genre_hash
    thread_pool = Array::new

    video_array.each do |group|
        # Create a download thread
        thread_temp = Thread.new {
            client = Javlibrary.client
            group.each do |item|
                begin
                    video_info_insert(client, item['video_num'], item['video_label'],
                        actor_hash, genre_hash)
                rescue
                    next
                end
            end
            client.close
        }
        thread_pool << thread_temp
    end
    thread_pool.map(&:join)
end

.download_all_video_label ⇒ Object 



267
268
269
270
271
272
273
274
275
276
 
# File 'lib/javlibrary.rb', line 267

def download_all_video_label
    thread_pool =[]
    'A'.upto('Z').each do |alphabet|
        thread_temp = Thread.new{
            select_actor(alphabet)
        }
        thread_pool << thread_temp
    end
    thread_pool.map(&:join)
end

.download_video_label(actor_id) ⇒ Object 



212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
 
# File 'lib/javlibrary.rb', line 212

def Javlibrary.download_video_label(actor_id)
    firsturl = "http://www.jav11b.com/ja/vl_star.php?s=#{actor_id}"
    baseurl = "http://www.jav11b.com/ja/vl_star.php?&mode=&s=#{actor_id}&page="

    begin
        response = RestClient.get firsturl
    rescue
        retry
    end

    doc = Nokogiri::HTML(response.body)
    last_page = 1
    doc.search('//div[@class="page_selector"]/a[@class="page last"]').each do |row|
        last_page = row['href'].split("=")[-1].to_i
    end

    result = []
    1.upto(last_page) do |page|
        tempurl = baseurl + page.to_s
        begin
            response = RestClient.get tempurl
        rescue
            retry
        end

        Nokogiri::HTML(response.body).search('//div[@class="video"]/a').each do |row|
            # Data:
            # Video_label: row['href'].split("=")[-1]
            # Video_title: row['title']
            # client.query("INSERT INTO label (lable) VALUES ('#{row['href'].split("=")[-1]}')")
            result << row['href'].split("=")[-1]
        end
    end

    client = Javlibrary.client
    result.each do |e|
        begin
            client.query("INSERT INTO label (video_label, video_download) VALUES ('#{e}', '0')")
        rescue
            next
        end
    end
    client.close
end

.downloader(identifer) ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/javlibrary.rb', line 17

def Javlibrary.downloader(identifer)
    baseurl = "http://www.jav11b.com/cn/?v=#{identifer}"
    response = Mechanize.new
    response.user_agent = Mechanize::AGENT_ALIASES.values[rand(21)]
    begin
        response.get baseurl
    rescue
        retry
    end

    doc = Nokogiri::HTML(response.page.body)

    video_title, details, video_genres, video_jacket_img = String.new, Array.new, String.new, String.new

    video_title = doc.search('div[@id="video_title"]/h3/a').children.text
    doc.search('//div[@id="video_info"]/div[@class="item"]/table/tr/td[@class="text"]').map do |row|
        details << row.children.text
    end

    doc.search('//div[@id="video_genres"]/table/tr/td[@class="text"]/span[@class="genre"]/a').each do |row|
        video_genres << row.children.text << " "
    end

    doc.search('//img[@id="video_jacket_img"]').each do |row|
        video_jacket_img = row['src']
    end

    # return data format: title$id$date$director$maker$label$cast$genres$img_url
    "#{video_title}$#{details[0]}$#{details[1]}$#{details[2]}$#{details[3]}$#{details[4]}$#{details[-1]}$#{video_genres}$#{video_jacket_img}"
    #result = Hash.new
    #result["title"] = video_title; result["id"] = details[0]; result["date"] = details[1]
    #result["director"] = details[2]; result["maker"] = details[3]; result["label"] = details[4]
    #result["cast"] = details[-1]; result["genres"] = video_genres; result["img_url"] = video_jacket_img
end

.genre_hash ⇒ Object 



124
125
126
127
128
129
130
131
132
133
 
# File 'lib/javlibrary.rb', line 124

def genre_hash
    client = Javlibrary.client
    category_hash = Hash.new
    client.query("SELECT * FROM category").each do |item|
        category_hash["#{item['category_name']}"] = item['category_id']
    end
    client.close

    category_hash
end

.genres ⇒ Object 



135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/javlibrary.rb', line 135

def Javlibrary.genres
    response = Mechanize.new; genres = Array.new
    begin
        response.get "http://www.jav11b.com/cn/genres.php"
    rescue
        retry
    end

    Nokogiri::HTML(response.page.body).search('//div[@class="genreitem"]/a').each do |row|
        genres << row.children.text
    end
    genres.uniq
end

.genres_insert ⇒ Object 



149
150
151
152
153
154
155
156
157
158
159
160
161
 
# File 'lib/javlibrary.rb', line 149

def genres_insert
    client = Javlibrary.client
    genres = genres()
    genres.each do |e|
        begin
            client.query("INSERT INTO category (category_name) VALUES ('#{e}')")
        rescue
            next
        end
    end

    client.close
end

.get_all_actor ⇒ Object 



171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/javlibrary.rb', line 171

def get_all_actor
    firsturl = "http://www.jav11b.com/cn/star_list.php?prefix="

    client = Javlibrary.client
    'A'.upto('Z') do |alphabet|
        tempurl = firsturl + alphabet
        begin
            response = RestClient.get tempurl
        rescue
            retry
        end

        doc = Nokogiri::HTML(response.body)
        last_page = author_page_num(doc)

        1.upto(last_page) do |page_num|
            temp_page_url = tempurl + "&page=#{page_num.to_s}"
            begin
                response_page = RestClient.get temp_page_url
            rescue
                retry
            end

            doc_page = Nokogiri::HTML(response_page.body)
            doc_page.search('//div[@class="starbox"]/div[@class="searchitem"]/a').each do |row|
                # row.text Actor.name
                # row['href'].split("=")[-1] Actor.label
                name = row.text; label = row['href'].split("=")[-1]
                begin
                    client.query("INSERT INTO actor (actor_name, actor_label, type)
                        VALUES ('#{name}', '#{label}', '#{alphabet}')")
                rescue
                    next
                end
            end
        end
    end

    client.close
end

.select_actor(type) ⇒ Object 



257
258
259
260
261
262
263
264
265
 
# File 'lib/javlibrary.rb', line 257

def Javlibrary.select_actor(type)
    client = Javlibrary.client
    result = client.query("SELECT actor_label FROM actor WHERE type='#{type}'")
    client.close

    result.each do |e|
        download_video_label(e["actor_label"])
    end
end

.test ⇒ Object 



108
109
110
 
# File 'lib/javlibrary/downloader.rb', line 108

def test
    pp downloader('javlia322m')
end

.video_info_insert(client, index, identifer, actor_hash, genres_hash) ⇒ Object 



52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/javlibrary.rb', line 52

def Javlibrary.video_info_insert(client, index, identifer, actor_hash, genres_hash)
    result = downloader(identifer)
    title, id, date, director, maker, label, cast_tmp, genres_tmp, img_url = downloader(identifer).split('$')
    cast = cast_tmp.split.reject(&:empty?)
    genres = genres_tmp.split.reject(&:empty?)
    begin
        client.query("INSERT INTO video (video_id,video_name,license,url,director,label,date,maker)
        VALUES (#{index},'#{title}','#{id}','#{img_url}','#{director}','#{label}','#{date}','#{maker}')")
    rescue
        return
    end
    cast.each do |a|
        a_tmp = actor_hash[a]
        next if a_tmp == nil
        client.query("INSERT INTO v2a (v2a_fk_video,v2a_fk_actor) VALUES(#{index}, #{a_tmp.to_i})")
    end

    genres.each do |g|
        g_tmp = genres_hash[g]
        next if g_tmp == nil
        client.query("INSERT INTO v2c (v2c_fk_video,v2c_fk_category) VALUES(#{index}, #{g_tmp.to_i})")
    end

    client.query("UPDATE label SET video_download=1 WHERE video_num=#{index}")
    return nil
end

Instance Method Details

#download_all_video_info ⇒ Object 



74
75
76
 
# File 'lib/javlibrary/video.rb', line 74

def download_all_video_info

end