Class: RightScriptSync::Downloader

Inherits:
Object
  • Object
show all
Defined in:
lib/rightscript_sync/downloader.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options) ⇒ Downloader

Returns a new instance of Downloader.



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/rightscript_sync/downloader.rb', line 33

def initialize(options)
  @log = Logger.new(STDOUT)
  @log.level = options[:log_level]
  @account_id = options[:account_id]
  @username = options[:username]
  @password = options[:password]
  @output_path = options[:output_path]
  @dry_run = options[:dry_run] || false
  @api_uri = "https://my.rightscale.com/api/acct/#{@account_id}"
  @site_uri = "https://my.rightscale.com/acct/#{@account_id}"

  @log.debug("account_id:#{@account_id} username:#{@username} password:#{@password}")
  @cookie_jar = Mechanize::CookieJar.new
  @agent = Mechanize.new do |agent|
    #agent.log = @log
    agent.user_agent_alias = 'Mac Safari'
    agent.verify_mode = ::OpenSSL::SSL::VERIFY_NONE
    agent.cookie_jar = @cookie_jar
  end
end

Instance Attribute Details

#account_id ⇒ Object

Returns the value of attribute account_id.



31
32
33
 
# File 'lib/rightscript_sync/downloader.rb', line 31

def 
  @account_id
end

#agent ⇒ Object

Returns the value of attribute agent.



31
32
33
 
# File 'lib/rightscript_sync/downloader.rb', line 31

def agent
  @agent
end

#cookie_jar ⇒ Object

Returns the value of attribute cookie_jar.



31
32
33
 
# File 'lib/rightscript_sync/downloader.rb', line 31

def cookie_jar
  @cookie_jar
end

#dry_run ⇒ Object

Returns the value of attribute dry_run.



31
32
33
 
# File 'lib/rightscript_sync/downloader.rb', line 31

def dry_run
  @dry_run
end

#log ⇒ Object

Returns the value of attribute log.



31
32
33
 
# File 'lib/rightscript_sync/downloader.rb', line 31

def log
  @log
end

#output_path ⇒ Object

Returns the value of attribute output_path.



31
32
33
 
# File 'lib/rightscript_sync/downloader.rb', line 31

def output_path
  @output_path
end

#password ⇒ Object

Returns the value of attribute password.



31
32
33
 
# File 'lib/rightscript_sync/downloader.rb', line 31

def password
  @password
end

#username ⇒ Object

Returns the value of attribute username.



31
32
33
 
# File 'lib/rightscript_sync/downloader.rb', line 31

def username
  @username
end

Instance Method Details

#download(url, headers) ⇒ Object 



127
128
129
130
131
132
 
# File 'lib/rightscript_sync/downloader.rb', line 127

def download(url, headers)
  @log.debug("Downloading #{url}")
  @agent.get(url, nil, nil, headers) do |page|
    return page.body
  end
end

#download_file(url, headers, file) ⇒ Object 



120
121
122
123
124
125
 
# File 'lib/rightscript_sync/downloader.rb', line 120

def download_file(url, headers, file)
 @log.debug("Downloading #{url} to '#{file}'")
 return if @dry_run
 mkbasedir(file)
 @agent.get(url, nil, nil, headers).save(file)
end

#download_right_script_attachments(right_script_id) ⇒ Object 



111
112
113
114
115
116
117
118
 
# File 'lib/rightscript_sync/downloader.rb', line 111

def download_right_script_attachments(right_script_id)
  @log.info("Downloading attachments for #{right_script_id}")
  headers = { 'X-Requested-With' => 'XMLHttpRequest' }
  url = "#{@site_uri}/right_scripts/#{right_script_id}/script_attachments"
  html = download(url, headers)
  #html = File.open("foo.html", "rb").read
  parse_right_script_attachments(html)
end

#download_right_scripts ⇒ Object 



102
103
104
105
106
107
108
109
 
# File 'lib/rightscript_sync/downloader.rb', line 102

def download_right_scripts
  @log.info("Downloading RightScripts")
  headers = { 'X-API-VERSION' => '1.0' }
  url = "#{@api_uri}/right_scripts.xml"
  xml = download(url, headers)
  #xml = File.open("right_scripts.xml", "rb").read
  parse_right_scripts(xml)
end

#execute ⇒ Object 



208
209
210
211
212
213
214
215
216
217
218
 
# File 'lib/rightscript_sync/downloader.rb', line 208

def execute
  begin
    
    store_right_scripts      
  rescue Interrupt
    exit
  rescue Exception => e
    puts e.message
    puts e.backtrace.join("\n")
  end
end

#login ⇒ Object 



94
95
96
97
98
99
100
 
# File 'lib/rightscript_sync/downloader.rb', line 94

def 
  @log.info("Logging in")
  headers = { 'X-API-VERSION' => '1.0' }
  headers['Authorization'] = 'Basic ' + Base64.encode64( @username + ':' + @password )
  url = "#{@api_uri}/login?api_version=1.0"
  download(url, headers)
end

#mkbasedir(file) ⇒ Object 



134
135
136
137
138
139
140
141
 
# File 'lib/rightscript_sync/downloader.rb', line 134

def mkbasedir(file)
  dirname = File.dirname(file)
  unless File.directory?(dirname)
    @log.debug("Creating directory '#{dirname}'")
    return if @dry_run
    FileUtils.mkdir_p(dirname) 
  end
end

#normalize_right_script_name(right_script_name) ⇒ Object 



204
205
206
 
# File 'lib/rightscript_sync/downloader.rb', line 204

def normalize_right_script_name(right_script_name)
  right_script_name.gsub(/[^A-Za-z0-9_\.]+/, '_').downcase
end

#parse_right_script_attachments(html) ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/rightscript_sync/downloader.rb', line 75

def parse_right_script_attachments(html)
  doc = Nokogiri::HTML(html)
  doc.encoding = 'UTF-8'
  right_script_attachments = [] 
  xpath_right_script_attachments = doc.xpath('//table[@id="right_scripts_show_script_attachments"]/tbody/tr')
  xpath_right_script_attachments.each do |xpath_right_script_attachment|
    right_script_attachment = {}
    right_script_attachment[:filename] = xpath_right_script_attachment.xpath('td[@data-column_name="Filename"]/a/text()').to_s.strip
    next if right_script_attachment[:filename].nil? || right_script_attachment[:filename].empty?
    right_script_attachment[:uri] = xpath_right_script_attachment.xpath('td[@data-column_name="Filename"]/a/@href').to_s.strip
    right_script_attachment[:size] = xpath_right_script_attachment.xpath('td[@data-column_name="Size"]/text()').to_s.strip
    right_script_attachment[:created_at] = xpath_right_script_attachment.xpath('td[@data-column_name="Created At"]/text()').to_s.strip
    right_script_attachment[:updated_at] = xpath_right_script_attachment.xpath('td[@data-column_name="Updated At"]/text()').to_s.strip
    right_script_attachment[:md5sum] = xpath_right_script_attachment.xpath('td[@data-column_name="md5sum"]/text()').to_s.strip
    right_script_attachments << right_script_attachment
  end
  return right_script_attachments
end

#parse_right_scripts(xml) ⇒ Object 



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/rightscript_sync/downloader.rb', line 54

def parse_right_scripts(xml)
  doc = Nokogiri::XML(xml)
  doc.encoding = 'UTF-8'
  right_scripts = [] 
  xpath_right_scripts = doc.xpath('/right-scripts[@type="array"]/right-script')
  xpath_right_scripts.each do |xpath_right_script|
    right_script = {}
    right_script[:name] = xpath_right_script.at_xpath('name/text()').to_s.strip
    right_script[:updated_at] = xpath_right_script.at_xpath('updated-at/text()').to_s.strip
    right_script[:created_at] = xpath_right_script.at_xpath('created-at/text()').to_s.strip
    right_script[:is_head_version] = xpath_right_script.at_xpath('is-head-version/text()').to_s.strip
    right_script[:href] = xpath_right_script.at_xpath('href/text()').to_s.strip
    right_script[:id] = right_script[:href].gsub(/^.*\//, '').to_i
    right_script[:version] = xpath_right_script.at_xpath('version/text()').to_s.strip.to_i
    right_script[:script] = xpath_right_script.at_xpath('script/text()').to_s.strip
    right_script[:description] = xpath_right_script.at_xpath('description/text()').to_s.strip
    right_scripts << right_script
  end
  return right_scripts
end

#store_file(file, data) ⇒ Object 



151
152
153
154
155
156
157
 
# File 'lib/rightscript_sync/downloader.rb', line 151

def store_file(file, data)
  return if @dry_run
  mkbasedir(file)
  File.open(file, 'w') do |fh|
    fh.write(data)
  end
end

#store_metadata(file, data) ⇒ Object 



143
144
145
146
147
148
149
 
# File 'lib/rightscript_sync/downloader.rb', line 143

def store_metadata(file, data)
  return if @dry_run
  mkbasedir(file)
  File.open(file, 'w') do |fh|
    YAML.dump(data, fh)
  end
end

#store_right_script(right_script) ⇒ Object 



183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/rightscript_sync/downloader.rb', line 183

def store_right_script(right_script)
  @log.info("Storing RightScript (#{right_script[:name]})")
  right_script_path = "#{@output_path}/#{right_script[:id]}/#{normalize_right_script_name(right_script[:name])}/#{right_script[:version]}"

  right_script_file_path = "#{right_script_path}/script.txt"
  store_file(right_script_file_path, right_script[:script])
  File.utime(Time.parse(right_script[:updated_at]), Time.parse(right_script[:created_at]), right_script_file_path) unless @dry_run

  right_script_metadata_path = "#{right_script_path}/metadata.yml"
  store_metadata(right_script_metadata_path, right_script)

  store_right_script_attachments(right_script, right_script_path)
end

#store_right_script_attachment(right_script_attachment, right_script_attachment_path) ⇒ Object 



159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/rightscript_sync/downloader.rb', line 159

def store_right_script_attachment(right_script_attachment, right_script_attachment_path)
  @log.info("Storing RightScript attachment '#{right_script_attachment[:filename]}' (#{right_script_attachment[:size]}) #{right_script_attachment[:updated_at]}")
  right_script_attachment_file_path = right_script_attachment_path + '/' + right_script_attachment[:filename]
  headers = {}
  url = right_script_attachment[:uri]
  if File.exists?(right_script_attachment_file_path)
    if Digest::MD5.file(right_script_attachment_file_path) == right_script_attachment[:md5sum]
      @log.info("Already downloaded #{right_script_attachment_file_path} with #{right_script_attachment[:md5sum]} md5")
      return
    end
  end
  download_file(url, headers, right_script_attachment_file_path)
  File.utime(Time.parse(right_script_attachment[:updated_at]), Time.parse(right_script_attachment[:created_at]), right_script_attachment_file_path) unless @dry_run
  @log.info("Attachment stored to #{right_script_attachment_file_path}")
end

#store_right_script_attachments(right_script, right_script_path) ⇒ Object 



175
176
177
178
179
180
181
 
# File 'lib/rightscript_sync/downloader.rb', line 175

def store_right_script_attachments(right_script, right_script_path)
  right_script_attachment_path = "#{right_script_path}/attachments"
  @log.info("Storing RightScript attachments for '#{right_script[:name]}' to #{right_script_attachment_path}")
  download_right_script_attachments(right_script[:id]).each do |right_script_attachment|
    store_right_script_attachment(right_script_attachment, right_script_attachment_path)
  end
end

#store_right_scripts ⇒ Object 



197
198
199
200
201
202
 
# File 'lib/rightscript_sync/downloader.rb', line 197

def store_right_scripts
  @log.info("Storing RightScripts")
  download_right_scripts.each do |right_script|
    store_right_script(right_script)
  end
end