Class: CloudFlock::Interface::CLI::App::Files

Inherits:
Object
  • Object
show all
Defined in:
lib/cloudflock/interface/cli/app/files.rb

Overview

Public: The Files app provides the interface to perform migrations of File/Object storage (e.g. Amazon S3, Local files and Rackspace Cloud Files).

Constant Summary collapse

CLI = 
CloudFlock::Interface::CLI::Console
DOWNLOAD_THREAD_COUNT = 
4
UPLOAD_THREAD_COUNT = 
4

Instance Method Summary collapse

Constructor Details

#initialize(opts) ⇒ Files

Public: Begin Files migration on the command line

opts - Hash containing options mappings.



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/cloudflock/interface/cli/app/files.rb', line 19

def initialize(opts)
  @options = opts
  @download_finished = false
  @download_mutex = Mutex.new
  @upload_mutex = Mutex.new
  @download_list = []
  @upload_list = []

  source_store = define_store("source")
  destination_store = define_store("destination")

  @source_container = define_container(source_store, "source")
  @destination_container = define_container(destination_store, "destination",
                                           true)

  if perform_migration
    puts "#{CLI.bold}#{CLI.blue}*** Migration complete#{CLI.reset}\a"
  else
    puts "#{CLI.bold}#{CLI.red}*** Migration failed#{CLI.reset}\a"
  end
rescue Excon::Errors::Unauthorized => err
  puts "A provider has returned an Unauthorized error."
  puts err.inspect if @options[:verbose]
  exit 1
end

Instance Method Details

#define_container(store, desc, create = false) ⇒ Object

Internal: Obtain the name of a container.

store - Fog object pointing to a Fog::Storage object. desc - String containing a description for the container. create - Boolean value indicating whether to create the container.

Returns a Fog object pointing to the container. Raises ArgumentError if store isn’t a Fog::Storage object. Raises ArgumentError if desc isn’t a String.



160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
 
# File 'lib/cloudflock/interface/cli/app/files.rb', line 160

def define_container(store, desc, create=false)
  unless store.class.to_s =~ /^Fog::Storage/
    raise ArgumentError, "Fog Storage object expected"
  end
  unless desc.kind_of?(String)
    raise ArgumentError, "String expected"
  end

  if create
    container = CLI.prompt("#{desc} container name")
    return store.directories.create(key: container)
  else
    puts "Available containers:"
    puts store.directories.map(&:key)
    container = CLI.prompt("#{desc} container name",
                           valid_answers: store.directories.map(&:key))
    return store.directories.select { |i| i.key == container }[0]
  end
end

#define_store(desc) ⇒ Object

Internal: Ascertain the location for a data store.

desc - String containing a description for the file store.

Returns a Fog object pointing to the data store. Raises ArgumentError if desc isn’t a String.



128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/cloudflock/interface/cli/app/files.rb', line 128

def define_store(desc)
  unless desc.kind_of?(String)
    raise ArgumentError, "String expected"
  end
  store = {}
  store[:provider] = CLI.prompt("#{desc} provider (aws, local, rax)",
                                valid_answers: ["rax", "aws", "local"])
  case store[:provider]
  when 'rax'
    store[:provider] = 'Rackspace'
    store[:rackspace_username] = CLI.prompt("Rackspace username")
    store[:rackspace_api_key] = CLI.prompt("Rackspace API key")
  when 'aws'
    store[:provider] = 'AWS'
    store[:aws_access_key_id] = CLI.prompt("AWS Access Key ID")
    store[:aws_secret_access_key] = CLI.prompt("AWS secret access key")
  when 'local'
    store[:local_root] = CLI.prompt("#{desc} location")
  end

  CloudFlock::Remote::Files.connect(store)
end

#download_thread ⇒ Object

Internal: Create a new Thread to download objects from the source container.

Returns a Thread.



80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/cloudflock/interface/cli/app/files.rb', line 80

def download_thread
  Thread.new do
    file = nil
    until @download_list.empty?
      @download_mutex.synchronize do
        file = @download_list.pop
      end
      next if file.nil?
      # AWS stores directories as their own object
      next if file.content_length == 0 && file.key =~ /\/$/

      tmp = Tempfile.new(file.object_id.to_s)
      @source_container.files.get(file.key) do |data, rem, cl|
        tmp.syswrite(data)
      end
      tmp.flush
      tmp.rewind
      @upload_mutex.synchronize do
        @upload_list.push(body: tmp, key: file.key)
      end
    end
  end
end

#perform_migration ⇒ Object

Internal: Migrate objects from the source store to the destination store.

Returns a boolean value corresponding to whether the migration has completed successfully.



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/cloudflock/interface/cli/app/files.rb', line 49

def perform_migration
  download_threads = []
  upload_threads = []

  @source_container.files.each { |f| @download_list.push(f) }

  DOWNLOAD_THREAD_COUNT.times do
    download_threads << download_thread
  end
  UPLOAD_THREAD_COUNT.times do
    upload_threads << upload_thread
  end

  download_threads.each { |t| t.join }
  @download_finished = true
  upload_threads.each { |t| t.join }
  true
rescue => e
  if @options[:verbose]
    puts "#{CLI.bold}#{CLI.red}*** Error ***#{CLI.reset}"
    puts e.inspect
    puts e.backtrace
    puts
  end
  false
end

#upload_thread ⇒ Object

Internal: Create a new Thread to upload objects to the desination container.

Returns a Thread.



108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/cloudflock/interface/cli/app/files.rb', line 108

def upload_thread
  Thread.new do
    file = nil
    until @upload_list.empty? && @download_finished
      sleep 0.1
      @upload_mutex.synchronize do
        file = @upload_list.pop
      end
      next if file.nil?
      @destination_container.files.create(file)
    end
  end
end