Class: COS::Download

Inherits:
Checkpoint show all
Includes:
Logging
Defined in:
lib/cos/download.rb

Overview

大文件分片下载, 支持断点续传, 支持多线程Range Headers support in HTTP1.1(rfc2616)

Constant Summary collapse

PART_SIZE =

默认分块大小

5 * 1024 * 1024
READ_SIZE =

默认文件读取大小

16 * 1024

Constants included from Logging

Logging::DEFAULT_LOG_FILE, Logging::MAX_NUM_LOG, Logging::ROTATE_SIZE

Constants inherited from Checkpoint

Checkpoint::DEFAULT_THREADS

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Logging

#logger, set_logger

Methods included from Struct::Base::AttrHelper

#optional_attrs, #required_attrs

Constructor Details

#initialize(opts = {}) ⇒ Download

Returns a new instance of Download.



22
23
24
25
26
 
# File 'lib/cos/download.rb', line 22

def initialize(opts = {})
  super(opts)

  @cpt_file = options[:cpt_file] || "#{File.expand_path(file_store)}.cpt"
end

Instance Attribute Details

#cpt_file ⇒ Object

Returns the value of attribute cpt_file.



20
21
22
 
# File 'lib/cos/download.rb', line 20

def cpt_file
  @cpt_file
end

#session ⇒ Object

Returns the value of attribute session.



20
21
22
 
# File 'lib/cos/download.rb', line 20

def session
  @session
end

Instance Method Details

#checkpoint ⇒ Object

断点续传状态记录

Examples:

states = {
  :session => 'session',
  :file => 'file',
  :file_meta => {
    :sha1 => 'file sha1',
    :size => 10000,
  },
  :parts => [
    {:number => 1, :range => [0, 100], :done => false},
    {:number => 2, :range => [100, 200], :done => true}
  ],
  :sha1 => 'checkpoint file sha1'
}


83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/cos/download.rb', line 83

def checkpoint
  logger.debug("Make checkpoint, options[:disable_cpt]: #{options[:disable_cpt] == true}")

  parts = sync_get_all_parts
  states = {
      :session    => session,
      :file       => file_store,
      :file_meta  => @file_meta,
      :parts      => parts
  }

  done = parts.count { |p| p[:done] }

  # 下载进度回调
  if progress
    if done == 0 or parts.count == 0
      progress.call(0.to_f)
    else
      percent = done.to_f / parts.size
      progress.call(percent > 1 ? 1.to_f : percent)
    end
  end

  write_checkpoint(states, cpt_file) unless options[:disable_cpt]

  logger.debug("Download Parts #{done}/#{parts.size}")
end

#download ⇒ Object

开始下载



29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/cos/download.rb', line 29

def download
  logger.info("Begin download, file: #{file_store}, threads: #{@num_threads}")

  # 重建断点续传
  rebuild

  # 文件分片
  divide_parts if @parts.empty?

  # 未完成的片段
  @todo_parts = @parts.reject { |p| p[:done] }

  # 多线程下载
  (1..@num_threads).map do
    logger.debug("#{@num_threads} Threads Downloads")

    Thread.new do
      logger.debug("Create Thread #{Thread.current.object_id}")

      loop do
        # 获取下一个未下载的片段
        p = sync_get_todo_part
        break unless p

        # 下载片段
        download_part(p)
      end
    end
  end.map(&:join)

  # 完成下载, 合并文件
  complete

  unless finish?
    File.delete(file_store) if File.exist?(file_store)
    raise DownloadError, 'File downloaded sha1 not match, deleted!'
  end
end