Class: VMC::KNIFE::DataService

Inherits:
Object
  • Object
show all
Includes:
Interactive
Defined in:
lib/vmc_knife/vmc_knife.rb,
lib/vmc_knife/data_services.rb

Overview

Read/Write the JSON for a dataservice. Does not map the actual JSON into a new ruby object.

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(root, data, role_name) ⇒ DataService

Returns a new instance of DataService.



97
98
99
100
101
 
# File 'lib/vmc_knife/vmc_knife.rb', line 97

def initialize(root, data, role_name)
  @root = root
  @wrapped = data
  @role_name = role_name
end

Instance Attribute Details

#role_name ⇒ Object

Returns the value of attribute role_name.



96
97
98
 
# File 'lib/vmc_knife/vmc_knife.rb', line 96

def role_name
  @role_name
end

#root ⇒ Object

Returns the value of attribute root.



96
97
98
 
# File 'lib/vmc_knife/vmc_knife.rb', line 96

def root
  @root
end

#wrapped ⇒ Object

Returns the value of attribute wrapped.



96
97
98
 
# File 'lib/vmc_knife/vmc_knife.rb', line 96

def wrapped
  @wrapped
end

Instance Method Details

#apply_privileges(app_name = nil) ⇒ Object

Make sure that all users who can connect to the DB can also access the tables. This workarounds the privilege issue reported .… and was added to “my” branch of vcap’s services

Another workaround though really not perfect so it will stay in vmc_knife: the ownership of the SQL functions.



454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
 
# File 'lib/vmc_knife/data_services.rb', line 454

def apply_privileges(app_name=nil)
  return if credentials(app_name).nil?
  if is_postgresql()
    cmd_acl="GRANT CREATE ON SCHEMA PUBLIC TO PUBLIC;\
  GRANT ALL ON ALL TABLES IN SCHEMA PUBLIC TO PUBLIC;\
  GRANT ALL ON ALL FUNCTIONS IN SCHEMA PUBLIC TO PUBLIC;\
  GRANT ALL ON ALL SEQUENCES IN SCHEMA PUBLIC TO PUBLIC;\
  ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO PUBLIC;\
  ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO PUBLIC;\
  ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO PUBLIC;"
#          shell(cmd_acl,true)
    
    # reset the owner of the functions to the current user
    # when there is a 'privileged' app.
    app_name ||= @wrapped['director']['bound_app'] if @wrapped['director']
    if app_name
      cmd_select_fcts="SELECT pg_proc.proname FROM pg_catalog.pg_proc WHERE \
     pg_proc.pronamespace=(SELECT pg_namespace.oid FROM pg_catalog.pg_namespace WHERE pg_namespace.nspname = 'public') \
 AND pg_proc.proowner!=(SELECT oid FROM pg_roles WHERE rolname = 'postgres')"
      current_owner=credentials()['username']
      unless current_owner
        STDERR.puts "The application #{app_name} is not bound to the data-service #{name}; not applying the database privileges."
        return
      end
      fcts_name=shell(cmd_select_fcts,true,true)
      if fcts_name.empty?
        puts "No need to re-assign the ownership of the functions in #{credentials()['name']}; the DB does not define its own functions." if VMC::Cli::Config.trace
      else
        fcts = fcts_name.split("\n").collect do |line|
          line.strip!
          "'#{line}'"
        end.join(',')
        cmd_change_fcts_owner="UPDATE pg_catalog.pg_proc \
 SET proowner = (SELECT oid FROM pg_roles WHERE rolname = '#{current_owner}')\
 WHERE pg_proc.proname IN (#{fcts})"
        puts `sudo -u postgres psql --dbname #{credentials()['name']} -c \"#{cmd_change_fcts_owner}\" #{PSQL_RAW_RES_ARGS}`
      end
    end
  end
end

#credentials(app_name = nil) ⇒ Object

The credentials hash for this data-service param app_name The favored app usually bound to this data_service



239
240
241
242
243
244
245
246
247
 
# File 'lib/vmc_knife/data_services.rb', line 239

def credentials(app_name=nil)
  #bound_app is the name of an app bound to the dat-service and that credentials
  #should be used to access the data-service.
  #for example if an app creates large objects you will need to use
  #the credentials of that app to find the objects.
  app_name ||= @wrapped['director']['bound_app'] if @wrapped['director']
  @credentials ||= VMC::KNIFE.get_credentials(name(), app_name)
  @credentials
end

#drop(collection_or_table_names = nil) ⇒ Object 



521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
 
# File 'lib/vmc_knife/data_services.rb', line 521

def drop(collection_or_table_names=nil)
  if is_postgresql
    sel_tables = "SELECT table_name FROM information_schema.tables WHERE table_schema='public'"
    if collection_or_table_names
      sel_tables = "#{sel_tables} AND table_name LIKE '#{collection_or_table_names}'"
    end
    tables = shell(sel_tables, false, true)
    tables_arr = Array.new
    tables.split("\n").each do | line |
      line.strip!
      tables_arr << line if line
    end
    if tables_arr.size > 0 && ask("Delete the tables \"#{tables_arr.join(',')}\"?", :default => true)
      #let's create a file in case there are a lot of tables:
      file = Tempfile.new('droptables')
      begin
        File.open(file.path, 'w') do |f2|
          tables_arr.each do |table|
            f2.puts "DROP TABLE public.#{table} CASCADE;"
          end
        end
        puts shell(file.path)
      ensure
        file.unlink   # deletes the temp file
      end
    end
    puts "Vacuum orphaned large objects..."
    cmd = VMC::KNIFE.pg_connect_cmd(credentials(), 'vacuumlo')
    puts cmd
    puts `#{cmd}`
  elsif is_mongodb
    # generate the command file from the erb template.
    filter = ".*"
    filterIsNegated = "false";
    skipSystem = "true";
    if collection_or_table_names
      if collection_or_table_names.start_with?('!')
        filterIsNegated = "true";
        filter = collection_or_table_names[1..-1]
      else
        filter = collection_or_table_names
      end
    end
    # this command is applied to each collection.
    # the name of the variable is 'collection' as can bee seen in the erb file.
    cmd="collection.drop();"
    
    file = Tempfile.new('dropcollections')
    begin
      File.open(file.path, 'w') do |f2|
        template = ERB.new File.new("#{VMCKNIFE::ROOT_REL}/vmc_knife/mongo/mongo_cmd.js.erb").read, nil, "%"
        f2.puts template.result(binding)
      end
      puts shell(file.path)
    ensure
      file.unlink   # deletes the temp file
    end
    
    #TODO: iterate over the collections and drop them according to the filter.
    #raise "TODO: Unsupported operation 'drop' for the data-service #{name()}"
  else
    puts "Unsupported operation 'drop' for the data-service #{name()}"
  end
end

#export(app_name = nil, file = nil) ⇒ Object 



366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
 
# File 'lib/vmc_knife/data_services.rb', line 366

def export(app_name=nil,file=nil)
  if is_postgresql
    if file.nil?
      extension = @wrapped['director']['file_extension'] if @wrapped['director']
      file = "#{name()}.#{extension}"
    else
      unless File.exists?(File.dirname(file))
        raise "The output folder #{File.dirname(file)} does not exist."
      end
    end
    extension ||= "dump"
    archive_unzipped = "#{name()}.#{extension}"
    #archive_unzipped="#{name()}.sql" unless /\.sql$/ =~ extension
    `touch #{archive_unzipped}`
    unless File.exists? archive_unzipped
      raise "Unable to create the file #{archive_unzipped}"
    end
    `chmod o+w #{archive_unzipped}`
    puts "Exports the database #{credentials(app_name)['name']} in #{file}"
    #sudo -u postgres env PGPASSWORD=intalio DBNAME=intalio DUMPFILE=intalio_dump.sql pg_dump --format=p --file=$DUMPFILE --no-owner --clean --blobs --no-acl --oid --no-tablespaces $DBNAME
    #sudo -u postgres env PGPASSWORD=$PGPASSWORD DUMPFILE=$DUMPFILE pg_dump --format=p --file=$DUMPFILE --no-owner --clean --blobs --no-acl --oid --no-tablespaces $DBNAME

    cmd = VMC::KNIFE.pg_connect_cmd(credentials(app_name), 'pg_dump', false, "--format=c --file=#{archive_unzipped} --no-owner --clean --oids --blobs --no-acl --no-privileges --no-tablespaces")
    puts cmd
    puts `#{cmd}`
    
    unless File.exists? archive_unzipped
      raise "Unable to read the file #{archive_unzipped}"
    end
    `chmod o-w #{archive_unzipped}`
  elsif is_mongodb
    if file.nil?
      extension = @wrapped['director']['file_extension'] if @wrapped['director']
      extension ||= "bson.tar.gz"
      file = "#{name()}.#{extension}"
    end
    creds=credentials(app_name)
    puts "Exports the database #{creds['name']} in #{file}"
    #mongodump --host localhost:27017
    mongodump_exec=VMC::KNIFE.get_mongo_exec('mongodump')
    # see if we go through the filesystem or through the network:
    base_dir=VMC::KNIFE.get_mongodb_base_dir()
    instance_name=creds['name']
    dbpath=File.join(base_dir, instance_name, 'data')            
    mongod_lock=File.join(dbpath,'mongod.lock')
    hostname = creds['hostname']
    puts "looking at #{mongod_lock} exists? #{File.exists?(mongod_lock)} size #{File.size(mongod_lock)}" if File.exists?(mongod_lock)
    if (File.exists?(mongod_lock) && File.size(mongod_lock)>0) || (hostname != '127.0.0.1' &&  hostname != 'localhost')
      cmd = "#{mongodump_exec} -u #{creds['username']} -p #{creds['password']} --host #{creds['hostname']}:#{creds['port']} --db db"
    else
      cmd = "#{mongodump_exec} --dbpath #{dbpath}"
    end
    puts cmd
    puts `#{cmd}`
    archive_unzipped="dump"
  end
    
  
  # this produces a dump folder in the working directory.
  # let's zip it:
  if /\.zip$/ =~ file
    # just zip
    `zip -r #{file} #{archive_unzipped}`
  elsif /\.tar$/ =~ file
    # just tar
    `tar -cvf #{file} #{archive_unzipped}`
  else
    # tar-gzip by default
    `tar -czvf #{file} #{archive_unzipped}`
  end
  `rm -rf #{archive_unzipped}` if archive_unzipped != file
end

#import(app_name = nil, file = nil) ⇒ Object 



254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
 
# File 'lib/vmc_knife/data_services.rb', line 254

def import(app_name=nil,file=nil)
  file ||= @wrapped['director']['import_url'] if @wrapped['director']
  if file.nil?
    files = Dir.glob("#{name()}.*")
    raise "Unable to locate the database file to import." if files.empty?
    file = files.first
  end
  

  tmp_download_filename="_download_.zip"
  data_download_dir="#{ENV['HOME']}/vmc_knife_downloads/data_#{@wrapped['name']}"
  current_wd=Dir.pwd
  FileUtils.mkdir_p data_download_dir
  Dir.chdir(data_download_dir) do
    if file =~ /^https?:\/\// || file =~ /^ftp:\/\//
      basename = Pathname.new(URI.parse(file).path).basename.to_s
    else
      file=File.expand_path(file,current_wd)
      basename = File.basename(file).to_s
    end
    
    unless File.exists?(basename)
      Dir.foreach(Dir.pwd) { |f|
        File.delete(f) unless f == '.' || f == '..'
      }
      if file =~ /^https?:\/\// || file =~ /^ftp:\/\//
        wget_args = @wrapped['director']['wget_args']
        if wget_args.nil?
          wget_args_str = ""
        elsif wget_args.kind_of? Array
          wget_args_str = wget_args.join(' ')
        elsif wget_args.kind_of? String
          wget_args_str = wget_args
        end
        `wget #{wget_args_str} --output-document=#{basename} #{file}`
        if $? != 0
          File.delete("#{data_download_dir}/#{basename}")
          raise "Unable to successfully download #{file}"
        end
      else
        FileUtils.cp(file, basename)
      end
    end
    #unzip if necessary (in progress)
    is_unzipped=true
    p "unzip #{basename}"
    if /\.tgz$/ =~ basename || /\.tar\.gz$/ =~ basename
      `tar zxvf #{basename}`
    elsif /\.tar$/ =~ basename
      `tar xvf #{basename}`
    elsif /\.zip$/ =~ basename
      `unzip #{basename}`
    else
      is_unzipped=false
    end
    
    if is_unzipped
      #`rm #{basename}`
      files = Dir.glob(["*.sql", "*.dump"]) if is_postgresql
      files = Dir.glob("**/*.bson") if is_mongodb
      files ||= Dir.glob("*")
      raise "Can't find the db-dump file." if files.empty?
      file = files.first
    else
      file = basename
    end
    
    creds=credentials(app_name)
    
    if is_postgresql
      p "chmod o+w #{file}"
      `chmod o+w #{file}`
      if /\.sql$/ =~ file
        other_params="--file #{file} --quiet"
        cmd = VMC::KNIFE.pg_connect_cmd(creds, 'psql',as_admin=false, other_params)
        #`psql --dbname #{dbname} --file #{file} --clean --quiet --username #{rolename}`
      else
        other_params="--clean --no-acl --no-privileges --no-owner #{file}"
        cmd = VMC::KNIFE.pg_connect_cmd(creds, 'pg_restore',false, other_params)
        #`pg_restore --dbname=#{dbname} --username=#{username} --no-acl --no-privileges --no-owner #{file}`
      end
      puts cmd
      puts `#{cmd}`.strip
      `chmod o-w #{file}`
    elsif is_mongodb
      
      # see if we go through the filesystem to shrink or
      # if we are only interested in the data itself.
      base_dir=VMC::KNIFE.get_mongodb_base_dir()
      instance_name=creds['name']
      dbpath=File.join(base_dir, instance_name, 'data')            
      mongod_lock=File.join(dbpath,'mongod.lock')
      
      if File.exists?(mongod_lock) && File.size(mongod_lock)>0
        # the mongodb instance is currently working. connect to it and do the work.
        # in that case import the 'db' alone. don't do the 'admin'
        VMC::KNIFE.data_service_console(creds, File.dirname(file), false, 'mongorestore')
      else
        # the mongodb instance is not currently working
        # go directly on the filesystem
        `rm -rf #{dbpath}`
        `mkdir -p #{dbpath}`
        #sudo mongorestore --dbpath /var/lib/mongodb
        mongorestore_exec=VMC::KNIFE.get_mongo_exec('mongorestore')
        `#{mongorestore_exec} --dbpath #{dbpath} #{File.dirname(File.dirname(file))}`
      end
    else
      raise "Unsupported type of data-service. Postgresql and mongodb are the only supported services at the moment."
    end
  end
end

#is_mongodb ⇒ Object 



443
444
445
 
# File 'lib/vmc_knife/data_services.rb', line 443

def is_mongodb()
  credentials()['db'] != nil
end

#is_postgresql ⇒ Object 



439
440
441
 
# File 'lib/vmc_knife/data_services.rb', line 439

def is_postgresql()
  credentials()['db'] == nil
end

#log(target_folder) ⇒ Object 



117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/vmc_knife/vmc_knife.rb', line 117

def log(target_folder)
  if is_postgresql()
    postgres_log_folder = "/var/log/postgresql"
    pg_target_dir = "#{target_folder}/postgresql"
    FileUtils.mkdir(pg_target_dir)
    Dir.glob(File.join(postgres_log_folder, "*")).each do |f|
      fname="#{File.basename(f)}"
      FileUtils.cp(File.join(postgres_log_folder, fname), pg_target_dir)
    end
  end

  if is_mongodb()
    mongo_log_folder = "/var/vcap/services/mongodb/logs"
    mg_target_dir = "#{target_folder}/mongodb"
    FileUtils.mkdir(mg_target_dir)
    Dir.glob(File.join(mongo_log_folder, "*")).each do |f|
      fname="#{File.basename(f)}"
      FileUtils.cp_r(File.join(mongo_log_folder, fname), mg_target_dir)
    end

  end
end

#name ⇒ Object

returns the name of the service for cloudfoundry



103
104
105
 
# File 'lib/vmc_knife/vmc_knife.rb', line 103

def name()
  @wrapped['name']
end

#shell(commands_file = nil, as_admin = false, return_value = false) ⇒ Object

Connect to the mongo js shell or the psql shell.



250
251
252
 
# File 'lib/vmc_knife/data_services.rb', line 250

def shell(commands_file=nil,as_admin=false,return_value=false)
  VMC::KNIFE.data_service_console(credentials(),commands_file,as_admin,nil,return_value)
end

#shrink(collection_or_table_names = nil) ⇒ Object

shrink the size of the databses on the file system. Specifically act on the mongodb instances when they are stopped.



497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
 
# File 'lib/vmc_knife/data_services.rb', line 497

def shrink(collection_or_table_names=nil)
  return unless is_mongodb
  creds=credentials()
  base_dir=VMC::KNIFE.get_mongodb_base_dir()
  instance_name=creds['name']
  dbpath=File.join(base_dir, instance_name, 'data')            
  mongod_lock=File.join(dbpath,'mongod.lock')
  raise "Can't shrink #{name}; the mongodb is currently running" if File.exists?(mongod_lock) && File.size(mongod_lock)>0
  mongodump_exec=VMC::KNIFE.get_mongo_exec('mongodump')
  raise "Can't find mongodump" unless File.exist? mongodump_exec
  mongorestore_exec=VMC::KNIFE.get_mongo_exec('mongorestore')
  raise "Can't find mongorestore" unless File.exist? mongorestore_exec
  cmd = "#{mongodump_exec} --dbpath #{dbpath}"
  puts "#{cmd}"
  puts `#{cmd}`
  
  `rm -rf #{dbpath}`
  `mkdir #{dbpath}`
  cmd = "#{mongorestore_exec} --dbpath #{dbpath} dump/"
  puts "#{cmd}"
  puts `#{cmd}`
  `rm -rf dump`
end

#to_vcap_manifest ⇒ Object

Returns a vcap manifest that can be used to create a new data-service to vcap’s cloud_controller.



112
113
114
115
 
# File 'lib/vmc_knife/vmc_knife.rb', line 112

def to_vcap_manifest()
  #TODO
  @wrapped
end

#vendor ⇒ Object 



106
107
108
 
# File 'lib/vmc_knife/vmc_knife.rb', line 106

def vendor()
  @wrapped['vendor']
end