Class: Down::Wget

Inherits:
Backend show all
Defined in:
lib/down/wget.rb

Defined Under Namespace

Modules: DownloadedFile Classes: Command

Instance Method Summary collapse

Methods inherited from Backend

download, open

Constructor Details

#initialize(*arguments) ⇒ Wget

Returns a new instance of Wget.



18
19
20
21
22
23
24
25
26
 
# File 'lib/down/wget.rb', line 18

def initialize(*arguments)
  @arguments = [
    user_agent:      "Down/#{Down::VERSION}",
    max_redirect:    2,
    dns_timeout:     30,
    connect_timeout: 30,
    read_timeout:    30,
  ] + arguments
end

Instance Method Details

#download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, **options) ⇒ Object 



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/down/wget.rb', line 28

def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, **options)
  io = open(url, *args, **options, rewindable: false)

  content_length_proc.call(io.size) if content_length_proc && io.size

  if max_size && io.size && io.size > max_size
    raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
  end

  extname  = File.extname(URI(url).path)
  tempfile = Tempfile.new(["down-wget", extname], binmode: true)

  until io.eof?
    chunk = io.readpartial(nil, buffer ||= String.new)

    tempfile.write(chunk)

    progress_proc.call(tempfile.size) if progress_proc

    if max_size && tempfile.size > max_size
      raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
    end
  end

  tempfile.open # flush written content

  tempfile.extend Down::Wget::DownloadedFile
  tempfile.url     = url
  tempfile.headers = io.data[:headers]

  download_result(tempfile, destination)
rescue
  tempfile.close! if tempfile
  raise
ensure
  io.close if io
end

#open(url, *args, rewindable: true, **options) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/down/wget.rb', line 66

def open(url, *args, rewindable: true, **options)
  arguments = generate_command(url, *args, **options)

  command = Down::Wget::Command.execute(arguments)
  output  = Down::ChunkedIO.new(
    chunks:     command.enum_for(:output),
    on_close:   command.method(:terminate),
    rewindable: false,
  )

  # https://github.com/tmm1/http_parser.rb/issues/29#issuecomment-309976363
  header_string  = output.readpartial
  header_string << output.readpartial until header_string.include?("\r\n\r\n")
  header_string, first_chunk = header_string.split("\r\n\r\n", 2)

  parser = HTTP::Parser.new
  parser << header_string

  if parser.headers.nil?
    output.close
    raise Down::Error, "failed to parse response headers"
  end

  headers = parser.headers
  status  = parser.status_code

  content_length = headers["Content-Length"].to_i if headers["Content-Length"]
  charset        = headers["Content-Type"][/;\s*charset=([^;]+)/i, 1] if headers["Content-Type"]

  chunks = Enumerator.new do |yielder|
    yielder << first_chunk if first_chunk
    yielder << output.readpartial until output.eof?
  end

  Down::ChunkedIO.new(
    chunks:     chunks,
    size:       content_length,
    encoding:   charset,
    rewindable: rewindable,
    on_close:   output.method(:close),
    data:       { status: status, headers: headers },
  )
end