Class: Pulse::Downloader::Client

Inherits:
Object
  • Object
show all
Includes:
FileChecker, FileDownloader, WebPageParser
Defined in:
lib/pulse/downloader/client.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from FileDownloader

#compute_hash_of, #download, #fetch_save_paths

Methods included from FileChecker

#file_path_in_file_list?

Methods included from WebPageParser

#fetch_file_paths

Constructor Details

#initialize(url:, file_type:, scrape_images: false, save_data: false, save_path: '', read_from_save_path: false, traverse_folders: false, verify_ssl: true, headers: nil, drop_exitsing_files_in_path: false, save_and_dont_return: true, report_time: false, progress_bar: false) ⇒ Client

TODO: Validation TODO: Retry TODO: DNS TODO: lib/pulse/downloader/file_downloader.rb:13: warning: URI.escape is obsolete



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
# File 'lib/pulse/downloader/client.rb', line 35

def initialize(url:,
  file_type:,
  scrape_images: false,
  save_data: false,
  save_path: '',
  read_from_save_path: false,
  traverse_folders: false,
  verify_ssl: true,
  headers: nil,
  drop_exitsing_files_in_path: false,
  save_and_dont_return: true,
  report_time: false,
  progress_bar: false)

  @url = url
  @file_type = file_type
  @scrape_images = scrape_images
  @save_data = save_data
  @save_path = save_path
  @read_from_save_path = read_from_save_path
  @traverse_folders = traverse_folders
  @verify_ssl = verify_ssl
  @headers = headers
  @drop_exitsing_files_in_path = drop_exitsing_files_in_path
  @save_and_dont_return = save_and_dont_return
  @report_time = report_time
  @progress_bar = progress_bar

  @base_url = get_base_url
  @folder_urls = []
end

Instance Attribute Details

#base_url ⇒ Object (readonly)

Returns the value of attribute base_url.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def base_url
  @base_url
end

#drop_exitsing_files_in_path ⇒ Object (readonly)

Returns the value of attribute drop_exitsing_files_in_path.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def drop_exitsing_files_in_path
  @drop_exitsing_files_in_path
end

#end_time ⇒ Object (readonly)

Returns the value of attribute end_time.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def end_time
  @end_time
end

#file_paths ⇒ Object (readonly)

Returns the value of attribute file_paths.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def file_paths
  @file_paths
end

#file_type ⇒ Object (readonly)

Returns the value of attribute file_type.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def file_type
  @file_type
end

#folder_urls ⇒ Object (readonly)

Returns the value of attribute folder_urls.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def folder_urls
  @folder_urls
end

#headers ⇒ Object (readonly)

Returns the value of attribute headers.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def headers
  @headers
end

#progress_bar ⇒ Object (readonly)

Returns the value of attribute progress_bar.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def progress_bar
  @progress_bar
end

#read_from_save_path ⇒ Object (readonly)

Returns the value of attribute read_from_save_path.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def read_from_save_path
  @read_from_save_path
end

#report_time ⇒ Object (readonly)

Returns the value of attribute report_time.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def report_time
  @report_time
end

#save_and_dont_return ⇒ Object (readonly)

Returns the value of attribute save_and_dont_return.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def save_and_dont_return
  @save_and_dont_return
end

#save_data ⇒ Object (readonly)

Returns the value of attribute save_data.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def save_data
  @save_data
end

#save_path ⇒ Object (readonly)

Returns the value of attribute save_path.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def save_path
  @save_path
end

#scrape_images ⇒ Object (readonly)

Returns the value of attribute scrape_images.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def scrape_images
  @scrape_images
end

#start_time ⇒ Object (readonly)

Returns the value of attribute start_time.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def start_time
  @start_time
end

#traverse_folders ⇒ Object (readonly)

Returns the value of attribute traverse_folders.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def traverse_folders
  @traverse_folders
end

#url ⇒ Object (readonly)

Returns the value of attribute url.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def url
  @url
end

#verify_ssl ⇒ Object (readonly)

Returns the value of attribute verify_ssl.



9
10
11
 
# File 'lib/pulse/downloader/client.rb', line 9

def verify_ssl
  @verify_ssl
end

Instance Method Details

#call ⇒ Object 



71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/pulse/downloader/client.rb', line 71

def call
  return false unless valid?

  @file_paths = fetch_file_paths

  if @progress_bar
    @progress_bar = ::ProgressBar.new(file_paths.size)
  end

  file_paths.map do |file_path|
    download(file_path, @progress_bar) if save_data
    @progress_bar.increment!
  end
end

#call! ⇒ Object 



67
68
69
 
# File 'lib/pulse/downloader/client.rb', line 67

def call!
  call
end

#valid? ⇒ Boolean

Returns:

  • (Boolean) 


86
87
88
 
# File 'lib/pulse/downloader/client.rb', line 86

def valid?
  true # TODO
end