Module: Javlibrary

Defined in:
lib/javlibrary.rb,
lib/javlibrary/init.rb,
lib/javlibrary/name.rb,
lib/javlibrary/video.rb,
lib/javlibrary/version.rb,
lib/javlibrary/database.rb,
lib/javlibrary/info_hash.rb,
lib/javlibrary/downloader.rb

Constant Summary collapse

VERSION = 
"0.1.2"

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.actor_hash ⇒ Object 



4
5
6
7
8
9
10
11
12
13
 
# File 'lib/javlibrary/info_hash.rb', line 4

def actor_hash
    client = Javlibrary.client
    actor_hash = Hash.new
    client.query("SELECT * FROM actor").each do |item|
        actor_hash["#{item['actor_name']}"] = item['actor_id']
    end
    client.close

    actor_hash
end

.author_page_num(nokogiri_doc) ⇒ Object 



4
5
6
7
8
9
10
 
# File 'lib/javlibrary/name.rb', line 4

def Javlibrary.author_page_num(nokogiri_doc)
    last_page = 1
    nokogiri_doc.search('//div[@class="page_selector"]/a[@class="page last"]').each do |row|
        last_page = row['href'].split("=")[-1].to_i
    end
    last_page
end

.client ⇒ Object 



4
5
6
7
8
9
 
# File 'lib/javlibrary/database.rb', line 4

def client
    client = Mysql2::Client.new(:host => "127.0.0.1",
                                :username => "root",
                                :password => "XuHefeng",
                                :database => "javlibrary_new")
end

.download_all_video ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/javlibrary/downloader.rb', line 66

def download_all_video
    client = Javlibrary.client
    result = client.query("SELECT video_num, video_label FROM label WHERE video_download=0")
    client.close

    video_array = Array.new
    result.each do |item|
        video_array << item
    end

    video_array = video_array.each_slice(5000).to_a
    actor_hash = Javlibrary::actor_hash
    genre_hash = Javlibrary::genre_hash
    thread_pool = Array::new

    video_array.each do |group|
        # Create a download thread
        thread_temp = Thread.new {
            client = Javlibrary.client
            group.each do |item|
                begin
                    video_info_insert(client, item['video_num'], item['video_label'],
                        actor_hash, genre_hash)
                rescue
                    next
                end
            end
            client.close
        }
        thread_pool << thread_temp
    end
    thread_pool.map(&:join)
end

.download_all_video_label ⇒ Object 



54
55
56
57
58
59
60
61
62
63
 
# File 'lib/javlibrary/video.rb', line 54

def download_all_video_label
    thread_pool =[]
    'A'.upto('Z').each do |alphabet|
        thread_temp = Thread.new{
            select_actor(alphabet)
        }
        thread_pool << thread_temp
    end
    thread_pool.map(&:join)
end

.download_video_label(actor_id) ⇒ Object 



4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/javlibrary/video.rb', line 4

def Javlibrary.download_video_label(actor_id)
    firsturl = "http://www.jav11b.com/ja/vl_star.php?s=#{actor_id}"
    baseurl = "http://www.jav11b.com/ja/vl_star.php?&mode=&s=#{actor_id}&page="

    begin
        response = RestClient.get firsturl
    rescue
        retry
    end

    doc = Nokogiri::HTML(response.body)
    last_page = 1
    doc.search('//div[@class="page_selector"]/a[@class="page last"]').each do |row|
        last_page = row['href'].split("=")[-1].to_i
    end

    result = []
    1.upto(last_page) do |page|
        tempurl = baseurl + page.to_s
        response = RestClient.get tempurl
        Nokogiri::HTML(response.body).search('//div[@class="video"]/a').each do |row|
            # Data:
            # Video_label: row['href'].split("=")[-1]
            # Video_title: row['title']
            # client.query("INSERT INTO label (lable) VALUES ('#{row['href'].split("=")[-1]}')")
            result << row['href'].split("=")[-1]
        end
    end

    client = Javlibrary.client
    result.each do |e|
        begin
            client.query("INSERT INTO label (video_label, video_download) VALUES ('#{e}', '0')")
        rescue
            next
        end
    end
    client.close
end

.downloader(identifer) ⇒ Object 



4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/javlibrary/downloader.rb', line 4

def Javlibrary.downloader(identifer)
    baseurl = "http://www.jav11b.com/cn/?v=#{identifer}"
    response = Mechanize.new
    response.user_agent = Mechanize::AGENT_ALIASES.values[rand(21)]
    begin
        response.get baseurl
    rescue
        retry
    end

    doc = Nokogiri::HTML(response.page.body)

    video_title, details, video_genres, video_jacket_img = String.new, Array.new, String.new, String.new

    video_title = doc.search('div[@id="video_title"]/h3/a').children.text
    doc.search('//div[@id="video_info"]/div[@class="item"]/table/tr/td[@class="text"]').map do |row|
        details << row.children.text
    end

    doc.search('//div[@id="video_genres"]/table/tr/td[@class="text"]/span[@class="genre"]/a').each do |row|
        video_genres << row.children.text << " "
    end

    doc.search('//img[@id="video_jacket_img"]').each do |row|
        video_jacket_img = row['src']
    end

    # return data format: title$id$date$director$maker$label$cast$genres$img_url
    "#{video_title}$#{details[0]}$#{details[1]}$#{details[2]}$#{details[3]}$#{details[4]}$#{details[-1]}$#{video_genres}$#{video_jacket_img}"
    #result = Hash.new
    #result["title"] = video_title; result["id"] = details[0]; result["date"] = details[1]
    #result["director"] = details[2]; result["maker"] = details[3]; result["label"] = details[4]
    #result["cast"] = details[-1]; result["genres"] = video_genres; result["img_url"] = video_jacket_img
end

.genre_hash ⇒ Object 



15
16
17
18
19
20
21
22
23
24
 
# File 'lib/javlibrary/info_hash.rb', line 15

def genre_hash
    client = Javlibrary.client
    category_hash = Hash.new
    client.query("SELECT * FROM category").each do |item|
        category_hash["#{item['category_name']}"] = item['category_id']
    end
    client.close

    category_hash
end

.genres ⇒ Object 



4
5
6
7
8
9
10
11
12
13
14
15
16
 
# File 'lib/javlibrary/init.rb', line 4

def Javlibrary.genres
    response = Mechanize.new; genres = Array.new
    begin
        response.get "http://www.jav11b.com/cn/genres.php"
    rescue
        retry
    end

    Nokogiri::HTML(response.page.body).search('//div[@class="genreitem"]/a').each do |row|
        genres << row.children.text
    end
    genres.uniq
end

.genres_insert ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/javlibrary/init.rb', line 18

def genres_insert
    client = Javlibrary.client
    genres = genres()
    genres.each do |e|
        begin
            client.query("INSERT INTO category (category_name) VALUES ('#{e}')")
        rescue
            next
        end
    end

    client.close
end

.get_all_actor ⇒ Object 



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/javlibrary/name.rb', line 12

def get_all_actor
    firsturl = "http://www.jav11b.com/cn/star_list.php?prefix="

    client = Javlibrary.client
    'A'.upto('Z') do |alphabet|
        tempurl = firsturl + alphabet
        response = RestClient.get tempurl

        doc = Nokogiri::HTML(response.body)
        last_page = author_page_num(doc)

        1.upto(last_page) do |page_num|
            temp_page_url = tempurl + "&page=#{page_num.to_s}"
            response_page = RestClient.get temp_page_url
            doc_page = Nokogiri::HTML(response_page.body)
            doc_page.search('//div[@class="starbox"]/div[@class="searchitem"]/a').each do |row|
                # row.text Actor.name
                # row['href'].split("=")[-1] Actor.label
                name = row.text; label = row['href'].split("=")[-1]
                begin
                    client.query("INSERT INTO actor (actor_name, actor_label, type)
                        VALUES ('#{name}', '#{label}', '#{alphabet}')")
                rescue
                    next
                end
            end
        end
    end

    client.close
end

.select_actor(type) ⇒ Object 



44
45
46
47
48
49
50
51
52
 
# File 'lib/javlibrary/video.rb', line 44

def Javlibrary.select_actor(type)
    client = Javlibrary.client
    result = client.query("SELECT actor_label FROM actor WHERE type='#{type}'")
    client.close

    result.each do |e|
        download_video_label(e["actor_label"])
    end
end

.test ⇒ Object 



100
101
102
 
# File 'lib/javlibrary/downloader.rb', line 100

def test
    pp downloader('javlia322m')
end

.video_info_insert(client, index, identifer, actor_hash, genres_hash) ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/javlibrary/downloader.rb', line 39

def Javlibrary.video_info_insert(client, index, identifer, actor_hash, genres_hash)
    result = downloader(identifer)
    title, id, date, director, maker, label, cast_tmp, genres_tmp, img_url = downloader(identifer).split('$')
    cast = cast_tmp.split.reject(&:empty?)
    genres = genres_tmp.split.reject(&:empty?)
    begin
        client.query("INSERT INTO video (video_id,video_name,license,url,director,label,date,maker)
        VALUES (#{index},'#{title}','#{id}','#{img_url}','#{director}','#{label}','#{date}','#{maker}')")
    rescue
        return
    end
    cast.each do |a|
        a_tmp = actor_hash[a]
        next if a_tmp == nil
        client.query("INSERT INTO v2a (v2a_fk_video,v2a_fk_actor) VALUES(#{index}, #{a_tmp.to_i})")
    end

    genres.each do |g|
        g_tmp = genres_hash[g]
        next if g_tmp == nil
        client.query("INSERT INTO v2c (v2c_fk_video,v2c_fk_category) VALUES(#{index}, #{g_tmp.to_i})")
    end

    client.query("UPDATE label SET video_download=1 WHERE video_num=#{index}")
    return nil
end

Instance Method Details

#download_all_video_info ⇒ Object 



65
66
67
 
# File 'lib/javlibrary/video.rb', line 65

def download_all_video_info

end