Class: VCAP::Services::Base::Warden::Service

Inherits:
Object
  • Object
show all
Includes:
Utils, InstanceUtils
Defined in:
lib/base/warden/service.rb

Class Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from InstanceUtils

#bind_mount_request, #container_destroy, #container_info, #container_run_command, #container_running?, #container_spawn_command, #container_start, #container_stop, included, #limit_bandwidth, #limit_memory, #map_port

Methods included from Utils

included

Class Attribute Details

.bandwidth_per_second ⇒ Object (readonly)

Returns the value of attribute bandwidth_per_second.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def bandwidth_per_second
  @bandwidth_per_second
end

.base_dir ⇒ Object (readonly)

Returns the value of attribute base_dir.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def base_dir
  @base_dir
end

.bin_dir ⇒ Object (readonly)

Returns the value of attribute bin_dir.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def bin_dir
  @bin_dir
end

.common_dir ⇒ Object (readonly)

Returns the value of attribute common_dir.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def common_dir
  @common_dir
end

.image_dir ⇒ Object (readonly)

Returns the value of attribute image_dir.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def image_dir
  @image_dir
end

.in_memory_status ⇒ Object (readonly)

Returns the value of attribute in_memory_status.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def in_memory_status
  @in_memory_status
end

.log_dir ⇒ Object (readonly)

Returns the value of attribute log_dir.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def log_dir
  @log_dir
end

.logger ⇒ Object (readonly)

Returns the value of attribute logger.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def logger
  @logger
end

.m_failed_times ⇒ Object (readonly)

Returns the value of attribute m_failed_times.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def m_failed_times
  @m_failed_times
end

.max_disk ⇒ Object (readonly)

Returns the value of attribute max_disk.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def max_disk
  @max_disk
end

.max_memory ⇒ Object (readonly)

Returns the value of attribute max_memory.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def max_memory
  @max_memory
end

.memory_overhead ⇒ Object (readonly)

Returns the value of attribute memory_overhead.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def memory_overhead
  @memory_overhead
end

.quota ⇒ Object (readonly)

Returns the value of attribute quota.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def quota
  @quota
end

.rm_instance_dir_timeout ⇒ Object (readonly)

Returns the value of attribute rm_instance_dir_timeout.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def rm_instance_dir_timeout
  @rm_instance_dir_timeout
end

.service_port ⇒ Object (readonly)

Returns the value of attribute service_port.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def service_port
  @service_port
end

.service_start_timeout ⇒ Object (readonly)

Returns the value of attribute service_start_timeout.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def service_start_timeout
  @service_start_timeout
end

.service_status_timeout ⇒ Object (readonly)

Returns the value of attribute service_status_timeout.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def service_status_timeout
  @service_status_timeout
end

.warden_socket_path ⇒ Object (readonly)

Returns the value of attribute warden_socket_path.



53
54
55
 
# File 'lib/base/warden/service.rb', line 53

def warden_socket_path
  @warden_socket_path
end

Class Method Details

.define_im_properties(*args) ⇒ Object 



40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/base/warden/service.rb', line 40

def define_im_properties(*args)
  args.each do |prop|
    define_method("#{prop}=".to_sym) do |value|
      self.class.in_memory_status[self[:name]] ||= {}
      self.class.in_memory_status[self[:name]][prop] = value
    end

    define_method(prop) do
      self.class.in_memory_status[self[:name]] && self.class.in_memory_status[self[:name]][prop]
    end
  end
end

.init(options) ⇒ Object 



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/base/warden/service.rb', line 12

def init(options)
  @@options = options
  @base_dir = options[:base_dir]
  @log_dir = options[:service_log_dir]
  @common_dir = options[:service_common_dir]
  @bin_dir = options[:service_bin_dir]
  @image_dir = options[:image_dir]
  @logger = options[:logger]
  @max_disk = options[:max_disk]
  @max_memory = options[:max_memory]
  @memory_overhead = options[:memory_overhead]
  @quota = options[:filesystem_quota] || false
  @service_start_timeout = options[:service_start_timeout] || 3
  @service_status_timeout = options[:service_status_timeout] || 3
  @bandwidth_per_second = options[:bandwidth_per_second]
  @service_port = options[:service_port]
  @rm_instance_dir_timeout = options[:rm_instance_dir_timeout] || 10
  @m_failed_times = options[:m_failed_times] || 3
  @warden_socket_path = options[:warden_socket_path] || "/tmp/warden.sock"
  FileUtils.mkdir_p(File.dirname(options[:local_db].split(':')[1]))
  DataMapper.setup(:default, options[:local_db])
  DataMapper::auto_upgrade!
  FileUtils.mkdir_p(base_dir)
  FileUtils.mkdir_p(log_dir)
  FileUtils.mkdir_p(image_dir) if @image_dir
  @in_memory_status = {}
end

Instance Method Details

#bandwidth_limit ⇒ Object

Generally the node can use this default calculation method for bandwidth limitation



423
424
425
 
# File 'lib/base/warden/service.rb', line 423

def bandwidth_limit
  self.class.bandwidth_per_second
end

#base_dir ⇒ Object 



283
284
285
286
 
# File 'lib/base/warden/service.rb', line 283

def base_dir
  return File.join(self.class.base_dir, self[:name]) if self.class.base_dir
  ''
end

#base_dir? ⇒ Boolean

Returns:

  • (Boolean) 


297
298
299
 
# File 'lib/base/warden/service.rb', line 297

def base_dir?
  Dir.exists?(base_dir)
end

#bin_dir ⇒ Object 



317
318
319
 
# File 'lib/base/warden/service.rb', line 317

def bin_dir
  self.class.bin_dir[version]
end

#common_dir ⇒ Object 



321
322
323
 
# File 'lib/base/warden/service.rb', line 321

def common_dir
  self.class.common_dir
end

#data_dir ⇒ Object 



309
310
311
 
# File 'lib/base/warden/service.rb', line 309

def data_dir
  File.join(base_dir, "data")
end

#delete ⇒ Object

instance operation helper



180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
 
# File 'lib/base/warden/service.rb', line 180

def delete
  container_name = self[:container]
  name = self[:name]
  task "destroy record in local db" do
    destroy! if saved?
  end

  task "delete in-memory status" do
    self.class.in_memory_status.delete(name)
  end

  task "stop container when deleting service #{name}" do
    stop(container_name)
  end

  task "delete instance directories" do
    if self.class.quota
      self.class.sh("rm -f #{image_file}", {:block => false})
    end
    # delete service data directory could be slow, so increase the timeout
    self.class.sh("rm -rf #{base_dir} #{log_dir} #{util_dirs.join(' ')}", {:block => false, :timeout => self.class.rm_instance_dir_timeout})
  end
end

#finish_first_start? ⇒ Boolean

For some services the check methods in instance provision and restart are different, then they should override this method, otherwise the behavior is the same with first_start

Returns:

  • (Boolean) 


388
389
390
 
# File 'lib/base/warden/service.rb', line 388

def finish_first_start?
  finish_start?
end

#finish_start? ⇒ Boolean

Check where the service process finish starting, the node subclass should override this function.

Returns:

  • (Boolean) 


382
383
384
 
# File 'lib/base/warden/service.rb', line 382

def finish_start?
  true
end

#first_start_options ⇒ Object

It’s the same with start_options except “is_first_start” key by default, but can be different for some services between instance provision (first start) and restart (normal start), then the node subclass need override this function



374
375
376
377
378
 
# File 'lib/base/warden/service.rb', line 374

def first_start_options
  options = start_options
  options[:is_first_start] = true
  options
end

#image_file ⇒ Object

directory helper



278
279
280
281
 
# File 'lib/base/warden/service.rb', line 278

def image_file
  return File.join(self.class.image_dir, "#{self[:name]}.img") if self.class.image_dir
  ''
end

#image_file? ⇒ Boolean

Returns:

  • (Boolean) 


293
294
295
 
# File 'lib/base/warden/service.rb', line 293

def image_file?
  File.exists?(image_file)
end

#in_monitored? ⇒ Boolean

Returns:

  • (Boolean) 


62
63
64
 
# File 'lib/base/warden/service.rb', line 62

def in_monitored?
  !failed_times || failed_times <= self.class.m_failed_times
end

#log_dir ⇒ Object 



288
289
290
291
 
# File 'lib/base/warden/service.rb', line 288

def log_dir
  return File.join(self.class.log_dir, self[:name]) if self.class.log_dir
  ''
end

#log_dir? ⇒ Boolean

Returns:

  • (Boolean) 


301
302
303
 
# File 'lib/base/warden/service.rb', line 301

def log_dir?
  Dir.exists?(log_dir)
end

#logger ⇒ Object 



66
67
68
 
# File 'lib/base/warden/service.rb', line 66

def logger
  self.class.logger
end

#loop_resize ⇒ Object 



124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/base/warden/service.rb', line 124

def loop_resize
  loop_up = loop_setup?
  loop_setdown if loop_up
  self.class.sh "cp #{image_file} #{image_file}.bak"
  begin
    old_size = File.size(image_file)
    self.class.sh "resize2fs -f #{image_file} #{self.class.max_disk.to_i}M"
    logger.info("Service #{self[:name]} change loop file size from #{old_size / 1024 / 1024}M to #{self.class.max_disk}M")
  rescue => e
    # Revert image file to the backup if resize raise error
    self.class.sh "cp #{image_file}.bak #{image_file}"
    logger.error("Service #{self[:name]} revert image file for error #{e}")
  ensure
    self.class.sh "rm -f #{image_file}.bak"
  end
  loop_setup if loop_up
end

#loop_setdown ⇒ Object 



97
98
99
 
# File 'lib/base/warden/service.rb', line 97

def loop_setdown
  self.class.sh "umount #{base_dir}"
end

#loop_setup ⇒ Object 



101
102
103
104
105
 
# File 'lib/base/warden/service.rb', line 101

def loop_setup
  self.class.sh "mount -n -o loop #{image_file} #{base_dir}"
  # Set the dir owner back to the process user
  self.class.sh "chown -R #{Process.uid}:#{Process.gid} #{base_dir}"
end

#loop_setup? ⇒ Boolean

Returns:

  • (Boolean) 


107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/base/warden/service.rb', line 107

def loop_setup?
  mounted = false
  File.open("/proc/mounts", mode="r") do |f|
    f.each do |w|
      if Regexp.new(base_dir) =~ w
        mounted = true
        break
      end
    end
  end
  mounted
end

#memory_limit ⇒ Object

Generally the node can use this default calculation method for memory limitation



414
415
416
417
418
419
420
 
# File 'lib/base/warden/service.rb', line 414

def memory_limit
  if self.class.max_memory
    (self.class.max_memory + (self.class.memory_overhead || 0)).to_i
  else
    nil
  end
end

#migration_check ⇒ Object 



151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
 
# File 'lib/base/warden/service.rb', line 151

def migration_check
  # incase for bosh --recreate, which will delete log dir
  FileUtils.mkdir_p(base_dir) unless base_dir?
  FileUtils.mkdir_p(log_dir) unless log_dir?

  if image_file?
    loop_resize if need_loop_resize?
    unless loop_setup?
      # for case where VM rebooted
      logger.info("Service #{self[:name]} mounting data file")
      loop_setup
    end
  else
    if self.class.quota
      logger.warn("Service #{self[:name]} need migration to quota")
      to_loopfile
    end
  end
end

#need_loop_resize? ⇒ Boolean

Returns:

  • (Boolean) 


120
121
122
 
# File 'lib/base/warden/service.rb', line 120

def need_loop_resize?
  image_file? && File.size(image_file) != self.class.max_disk * 1024 * 1024
end

#prepare_filesystem(max_size, opts = {}) ⇒ Object 



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/base/warden/service.rb', line 70

def prepare_filesystem(max_size, opts={})
  if base_dir?
    self.class.sh "umount #{base_dir}", :raise => false if self.class.quota
    logger.warn("Service #{self[:name]} base_dir:#{base_dir} already exists, deleting it")
    FileUtils.rm_rf(base_dir)
  end
  if log_dir?
    logger.warn("Service #{self[:name]} log_dir:#{log_dir} already exists, deleting it")
    FileUtils.rm_rf(log_dir)
  end
  if image_file?
    logger.warn("Service #{self[:name]} image_file:#{image_file} already exists, deleting it")
    FileUtils.rm_f(image_file)
  end
  FileUtils.mkdir_p(base_dir)
  FileUtils.mkdir_p(data_dir)
  FileUtils.mkdir_p(log_dir)

  ext_opts = nil
  ext_opts = "-E \"lazy_itable_init=1\"" if opts[:lazy_itable_init]
  if self.class.quota
    self.class.sh "dd if=/dev/null of=#{image_file} bs=1M seek=#{max_size.to_i}"
    self.class.sh "mkfs.ext4 -q -F -O \"^has_journal,uninit_bg\" #{"#{ext_opts}" if ext_opts} #{image_file}"
    loop_setup
  end
end

#run(options = nil, &post_start_block) ⇒ Object

The logic in instance run function is:

  1. To avoid to create orphan, clean up container if handle exists

  2. Generate bind mount request and create warden container with bind mount options

  3. Limit memory and bandwidth of the container (optional)

  4. Run pre service start script (optional)

  5. Run service start script

  6. Create iptables rules for service process (optional)

  7. Get container IP address and wait for the service finishing starting

  8. Run post service start script (optional)

  9. Run post service start block (optional)

  10. Save the instance info to local db

Raises:



223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
 
# File 'lib/base/warden/service.rb', line 223

def run(options=nil, &post_start_block)
  stop if self[:container] && self[:container].length > 0
  # If no options specified, then check whether the instance is stored in local db
  # to decide to use which start options
  options = (new? ? first_start_options : start_options) unless options
  loop_setup if self.class.quota && (not loop_setup?)
  bind_mounts = []
  bind_mounts = options[:bind_dirs].map { |bind_dir| bind_mount_request(bind_dir) }
  handle = container_start(bind_mounts)
  self[:container] = handle
  rw_dirs = options[:bind_dirs].map { |bind_dir| bind_dir[:dst] || bind_dir[:src] unless bind_dir[:read_only] }.compact
  run_command(handle, {:script => "chown -R vcap:vcap #{rw_dirs.join(' ')}", :use_root => true}) unless rw_dirs.empty?
  limit_memory(handle, memory_limit) if memory_limit
  limit_bandwidth(handle, bandwidth_limit) if bandwidth_limit
  run_command(handle, options[:pre_start_script]) if options[:pre_start_script]
  run_command(handle, options[:start_script]) if options[:start_script]
  map_port(handle, self[:port], options[:service_port]) if options[:need_map_port]
  rsp = container_info(handle)
  self[:ip] = rsp.container_ip
  # Check whether the service finish starting,
  # the check method can be different depends on whether the service is first start
  raise VCAP::Services::Base::Error::ServiceError::new(VCAP::Services::Base::Error::ServiceError::SERVICE_START_TIMEOUT) unless wait_service_start(options[:service_start_timeout], options[:is_first_start])
  run_command(handle, options[:post_start_script]) if options[:post_start_script]
  # The post start block is some work that need do after first start in provision,
  # where restart this instance, there should be no such work
  post_start_block.call(self) if post_start_block
  save!
  true
end

#run_command(handle, cmd_hash) ⇒ Object 



204
205
206
207
208
209
210
 
# File 'lib/base/warden/service.rb', line 204

def run_command(handle, cmd_hash)
  if cmd_hash[:use_spawn]
    container_spawn_command(handle, cmd_hash[:script], cmd_hash[:use_root])
  else
    container_run_command(handle, cmd_hash[:script], cmd_hash[:use_root])
  end
end

#running? ⇒ Boolean

Returns:

  • (Boolean) 


253
254
255
 
# File 'lib/base/warden/service.rb', line 253

def running?
  finish_start?
end

#script_dir ⇒ Object 



313
314
315
 
# File 'lib/base/warden/service.rb', line 313

def script_dir
  File.join(self.class.common_dir, "bin")
end

#service_script ⇒ Object

Generally the node can use this default service script path



409
410
411
 
# File 'lib/base/warden/service.rb', line 409

def service_script
  File.join(script_dir, "warden_service_ctl")
end

#start_options ⇒ Object

Instance start options, basically the node need define “:start_script”, and use other default options.



351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
 
# File 'lib/base/warden/service.rb', line 351

def start_options
  bind_dirs = []
  bind_dirs << {:src => bin_dir, :read_only => true}
  bind_dirs << {:src => common_dir, :read_only => true}
  # Since the script "warden_service_ctl" writes log in a hard-code directory "/var/vcap/sys/log/monit,
  # then we need has this directory with write permission in warden container,
  # now the work around is bind-mount instance log dir to "/var/vcap/sys/log/monit"
  bind_dirs << {:src => log_dir, :dst => "/var/vcap/sys/log/monit"}
  bind_dirs << {:src => base_dir}
  bind_dirs << {:src => log_dir}
  bind_dirs.concat util_dirs.map { |dir| {:src => dir} }
  {
    :service_port => self.class.service_port,
    :need_map_port => true,
    :is_first_start => false,
    :bind_dirs => bind_dirs,
    :service_start_timeout => self.class.service_start_timeout,
  }
end

#status_options ⇒ Object

Provide a command to monitor the health of instance. if status_options is empty, running? method will only show the health of container



402
403
404
405
406
 
# File 'lib/base/warden/service.rb', line 402

def status_options
  {
    :status_script => {:script => "#{service_script} status #{base_dir} #{log_dir} #{common_dir}"}
  }
end

#stop(container_name = nil) ⇒ Object

Usually, stop can retrieve container_name from local_db An exception is unprovision, which destroys record in local_db first.



259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
 
# File 'lib/base/warden/service.rb', line 259

def stop(container_name=nil)
  name = container_name || self[:container]
  if container_running?(name)
    begin
      run_command(name, stop_options[:stop_script]) if stop_options[:stop_script]
    rescue => e
      logger.error("Failed to call instance stop script #{stop_options[:stop_script]} with error #{e}")
    end
    container_stop(name)
    container_destroy(name)
    unless container_name
      self[:container] = ''
      save
    end
    loop_setdown if self.class.quota
  end
end

#stop_options ⇒ Object

Base user should provide a script to stop instance. if stop_options is empty, the process will get a SIGTERM first then SIGKILL later.



394
395
396
397
398
 
# File 'lib/base/warden/service.rb', line 394

def stop_options
  {
    :stop_script => {:script => "#{service_script} stop #{base_dir} #{log_dir} #{common_dir}"},
  }
end

#task(desc) ⇒ Object 



171
172
173
174
175
176
177
 
# File 'lib/base/warden/service.rb', line 171

def task(desc)
  begin
    yield
  rescue => e
    logger.error("Fail to #{desc}. Error: #{e}")
  end
end

#to_loopfile ⇒ Object 



142
143
144
145
146
147
148
149
 
# File 'lib/base/warden/service.rb', line 142

def to_loopfile
  self.class.sh "mv #{base_dir} #{base_dir+"_bak"}"
  self.class.sh "mkdir -p #{base_dir}"
  self.class.sh "A=`du -sm #{base_dir+"_bak"} | awk '{ print $1 }'`;A=$((A+32));if [ $A -lt #{self.class.max_disk.to_i} ]; then A=#{self.class.max_disk.to_i}; fi;dd if=/dev/null of=#{image_file} bs=1M seek=$A"
  self.class.sh "mkfs.ext4 -q -F -O \"^has_journal,uninit_bg\" #{image_file}"
  self.class.sh "mount -n -o loop #{image_file} #{base_dir}"
  self.class.sh "cp -af #{base_dir+"_bak"}/* #{base_dir}", :timeout => 60.0
end

#update_bind_dirs(bind_dirs, old_bind, new_bind) ⇒ Object 



325
326
327
328
329
330
331
332
 
# File 'lib/base/warden/service.rb', line 325

def update_bind_dirs(bind_dirs, old_bind, new_bind)
  find_bind = bind_dirs.select { |bind| bind[:src] == old_bind[:src] && bind[:dst] == old_bind[:dst] && bind[:read_only] == old_bind[:read_only] }
  unless find_bind.empty?
    find_bind[0][:src] = new_bind[:src]
    find_bind[0][:dst] = new_bind[:dst]
    find_bind[0][:read_only] = new_bind[:read_only]
  end
end

#util_dirs ⇒ Object 



305
306
307
 
# File 'lib/base/warden/service.rb', line 305

def util_dirs
  []
end

#wait_service_start(service_start_timeout, is_first_start = false) ⇒ Object

service start/stop helper



335
336
337
338
339
340
341
342
343
344
345
 
# File 'lib/base/warden/service.rb', line 335

def wait_service_start(service_start_timeout, is_first_start=false)
  (service_start_timeout * 10).times do
    sleep 0.1
    if is_first_start
      return true if finish_first_start?
    else
      return true if finish_start?
    end
  end
  false
end