400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
|
# File 'lib/cnvrg/datafiles.rb', line 400
def upload_large_files_s3(upload_resp, file_path)
begin
sts_path = upload_resp["result"]["path_sts"]
retries = 0
success= false
while !success and retries < 20
begin
if !Helpers.is_verify_ssl
body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
else
body = open(sts_path).read
end
success = true
rescue => e
retries +=1
sleep(5)
end
end
if !success
return false
end
split = body.split("\n")
key = split[0]
iv = split[1]
is_s3 = upload_resp["result"]["is_s3"]
access = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
secret = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_s"])
session = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_st"])
region = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["region"])
bucket = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
server_side_encryption =upload_resp["result"]["server_side_encryption"]
use_accelerate_endpoint = false
if is_s3 or is_s3.nil?
use_accelerate_endpoint =true
client = Aws::S3::Client.new(
:access_key_id =>access,
:secret_access_key => secret,
:session_token => session,
:region => region,
:http_open_timeout => 60, :retry_limit => 20)
else
endpoint = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["endpoint"])
use_accelerate_endpoint = false
client = Aws::S3::Client.new(
:access_key_id =>access,
:secret_access_key => secret,
:region => region,
:endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
:http_open_timeout => 60, :retry_limit => 20)
end
if !server_side_encryption
options = {:use_accelerate_endpoint => use_accelerate_endpoint}
else
options = {:use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => server_side_encryption}
end
s3 = Aws::S3::Resource.new(client: client)
resp = s3.bucket(bucket).
object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
upload_file(file_path,options)
return resp
rescue => e
puts e.message
return false
end
return true
end
|