Module: Downspout

Defined in:
lib/downspout/base.rb,
lib/downspout/config.rb,
lib/downspout/version.rb,
lib/downspout/tmp_file.rb,
lib/downspout/credential.rb,
lib/downspout/downloader.rb

Defined Under Namespace

Classes: BadURL, Base, Config, Credential, Downloader, ExcessiveRedirects, Tmpfile, UnsupportedScheme

Constant Summary collapse

VERSION = 
"0.4.0"

Class Method Summary collapse

Class Method Details

.clean_download_dir(delay = 30) ⇒ Object

Utility method for periodically removing download files from the configured directory. Expects an integer as the number of minutes ‘old’ a file should be before removal.



97
98
99
 
# File 'lib/downspout/tmp_file.rb', line 97

def self.clean_download_dir( delay=30 )
  Tmpfile.clean_dir( Downspout::Config.tmp_dir, delay )
end

.download_url_to_path(the_url, the_path) ⇒ Object

:nodoc:



43
44
45
 
# File 'lib/downspout/base.rb', line 43

def self.download_url_to_path( the_url, the_path ) #:nodoc:
  return self.fetch_url( the_url, the_path )
end

.fetch_url(some_url, some_path = nil) ⇒ Object

Download a file from a given URL. The path is optional and will default to a generated temporary file.



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/downspout/base.rb', line 20

def self.fetch_url( some_url, some_path = nil )
  $logger.debug("downspout | fetch_url |  URL : #{some_url}")
  $logger.debug("downspout | fetch_url |  Download Path : #{some_path}")

  begin
    d = Downspout::Downloader.new( :url => some_url, :path => some_path )
  rescue Exception => e
    $logger.error("downspout | fetch_url | Exception : '#{e}'")
    return nil if e.class == Downspout::UnsupportedScheme
    raise e
  end

  fetched = d.download!

  if !(fetched) then
    $logger.error("downspout | fetch_url |  Fetch Failed : #{d.url} ")
    return nil
  end

  $logger.debug("downspout | fetch_url |  Local File : #{d.path} ")
  return d
end

.supported_protocol?(some_protocol) ⇒ Boolean

Utility method for checking the support for URLs of the given network protocol or ‘scheme’

Returns:

  • (Boolean) 


71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/downspout/base.rb', line 71

def self.supported_protocol?( some_protocol )
  $logger.debug("downspout | supported_protocol? |  protocol : #{some_protocol} ")

  protocol_string = some_protocol.to_s.upcase

  return true if self.supported_protocols.include?( protocol_string )

  case protocol_string
  when "HTTP"
    return true
  when "URI::HTTP"
    return true
  when "HTTPS"
    return true
  when "URI::HTTPS"
    return true
  when "FTP"
    return true
  when "URI::FTP"
    return true
  when "FTPS"
    return true
  when "URI::FTPS"
    return true
  else
    $logger.warn("downspout | supported_protocol? | #{protocol_string} is not supported by Downspout.")
  end

  return false
end

.viable_url?(url_string) ⇒ Boolean

Utility method for validating a URL without initiating a download

Returns:

  • (Boolean) 


50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/downspout/base.rb', line 50

def self.viable_url?( url_string )
  $logger.info("downspout | viable_url? |  URL : #{url_string} ")

  begin
    # remove user/password prefix if provided
    clean_url = self.extract_credentials_from_url!( url_string )

    uri = URI.parse( clean_url )
  rescue URI::InvalidURIError
    $logger.warn("downspout | viable_url? | The format of the url is not valid : #{url_string}")
    return false
  end

  return false unless self.supported_protocol?( uri.scheme )

  return true
end