Class: CropDuster::Importer

Inherits:
Object
  • Object
show all
Defined in:
lib/crop_duster/importer.rb

Constant Summary collapse

REGIONS = 
{
  aws: ['ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-1', 'sa-east-1', 'us-east-1', 'us-west-1', 'us-west-2']
}

Class Method Summary collapse

Class Method Details

.import ⇒ Object 



9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/crop_duster/importer.rb', line 9

def import
  import_id = SecureRandom.hex

  self.import_servers(import_id)

  if CropDuster.aws_billing_bucket.present?
    self.import_billing(import_id)
  else
    puts "No billing bucket provided, so precise AWS billing information is unknown."
  end
end

.import_billing(import_id = SecureRandom.hex) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/crop_duster/importer.rb', line 47

def import_billing(import_id = SecureRandom.hex)
  ar_conn = ActiveRecord::Base.connection_pool.checkout
  conn = ar_conn.raw_connection

  conn.exec("TRUNCATE aws_billing_imports")

  s = Fog::Storage.new CropDuster.aws_keys
  path = "#{CropDuster.tmp_dir}/#{import_id}"
  Dir.mkdir path

  s.directories.get(CropDuster.aws_billing_bucket).files.find_all { |f| f.key =~ /-aws-billing-detailed-line-items-with-resources-and-tags-/ }.each do |file|
    `curl -o #{path}/file.zip '#{file.url(15.minutes.since)}'`
    `unzip -p #{path}/file.zip > #{path}/#{Time.now.strftime('%Y%m%d%H%M%S.csv')}`
    `rm #{path}/file.zip`
  end

  Dir.new(path).each do |file|
    next if file =~ /^\./
    headers = nil

    conn.copy_data "COPY aws_billing_imports FROM STDIN CSV" do
      File.new(File.join(path, file)).each_line do |line|
        if headers.nil?
          headers = line.strip.split(/,/).map(&:underscore)
        else
          conn.put_copy_data line
        end
      end
    end
  end

  ActiveRecord::Base.connection_pool.checkin(ar_conn)

  `rm #{path}/*.csv`
end

.import_servers(import_id = SecureRandom.hex) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/crop_duster/importer.rb', line 21

def import_servers(import_id = SecureRandom.hex)
  REGIONS[:aws].each do |region|
    compute = Fog::Compute.new CropDuster.aws_keys.merge(region: region)

    compute.servers.each do |s|
      server = Server.import(s)
      server.update_attributes(last_import_id: import_id)
    end

    compute.volumes.each do |v|
      volume = Volume.import(v)
      volume.update_attributes(last_import_id: import_id)
    end

    #compute.snapshots.each do |hash|
      #snapshot = Snapshot.import(hash)
      #snapshot.update_attributes(last_import_id: import_id)
    #end

    #Snapshot.where("last_import_id != ? AND deleted_at IS NULL", import_id).update_all(deleted_at: Time.now)
  end

  Server.where("last_import_id != ? AND deleted_at IS NULL", import_id).update_all(deleted_at: Time.now)
  Volume.where("last_import_id != ? AND deleted_at IS NULL", import_id).update_all(deleted_at: Time.now)
end