Class: DataDuck::Commands

Inherits:

Object

• Object
• DataDuck::Commands

Defined in:

lib/dataduck/commands.rb

Defined Under Namespace

Classes: Namespace

Class Method Summary

• .acceptable_commands ⇒ Object
• .c ⇒ Object
• .console ⇒ Object
• .d(where = "destination") ⇒ Object
• .dbconsole(where = "destination") ⇒ Object
• .etl(what = nil) ⇒ Object
• .help ⇒ Object
• .prompt_choices(choices = []) ⇒ Object
• .quickstart ⇒ Object
• .quickstart_create_table(table_name, db) ⇒ Object
• .quickstart_register_email(email) ⇒ Object
• .quickstart_save_file(output_path_full, contents) ⇒ Object
• .quickstart_update_gitignore ⇒ Object
• .recreate(table_name) ⇒ Object
• .route_command(args) ⇒ Object
• .show(table_name = nil) ⇒ Object

Class Method Details

.acceptable_commands ⇒ Object

# File 'lib/dataduck/commands.rb', line 35

def self.acceptable_commands
  ['c', 'console', 'd', 'dbconsole', 'etl', 'quickstart', 'recreate', 'show']
end

.c ⇒ Object

# File 'lib/dataduck/commands.rb', line 53

def self.c
  self.console
end

.console ⇒ Object

# File 'lib/dataduck/commands.rb', line 57

def self.console
  require "irb"
  ARGV.clear
  IRB.start
end

.d(where = "destination") ⇒ Object

# File 'lib/dataduck/commands.rb', line 63

def self.d(where = "destination")
  self.dbconsole(where)
end

.dbconsole(where = "destination") ⇒ Object

# File 'lib/dataduck/commands.rb', line 67

def self.dbconsole(where = "destination")
  which_database = nil
  if where == "destination"
    which_database = DataDuck::Destination.only_destination
  elsif where == "source"
    which_database = DataDuck::Source.only_source
  else
    found_source = DataDuck::Source.source(where, true)
    found_destination = DataDuck::Destination.destination(where, true)
    if found_source && found_destination
      raise ArgumentError.new("Ambiguous call to dbconsole for #{ where } since there is both a source and destination named #{ where }.")
    end

    which_database = found_source if found_source
    which_database = found_destination if found_destination
  end

  if which_database.nil?
    raise ArgumentError.new("Could not find database '#{ where }'")
  end

  puts "Connecting to #{ where }..."
  which_database.dbconsole
end

.etl(what = nil) ⇒ Object

# File 'lib/dataduck/commands.rb', line 92

def self.etl(what = nil)
  if what.nil?
    puts "You need to specify a table name or 'all'. Usage: dataduck etl all OR datduck etl my_table_name"
    return
  end

  only_destination = DataDuck::Destination.only_destination

  if what == "all"
    etl = ETL.new(destinations: [only_destination], autoload_tables: true)
    etl.process!
  else
    table_name_camelized = DataDuck::Util.underscore_to_camelcase(what)
    require DataDuck.project_root + "/src/tables/#{ what }.rb"
    table_class = Object.const_get(table_name_camelized)
    if !(table_class <= DataDuck::Table)
      raise Exception.new("Table class #{ table_name_camelized } must inherit from DataDuck::Table")
    end

    table = table_class.new
    etl = ETL.new(destinations: [only_destination], autoload_tables: false, tables: [table])
    etl.process_table!(table)
  end
end

.help ⇒ Object

# File 'lib/dataduck/commands.rb', line 117

def self.help
  puts "Usage: dataduck commandname"
  puts "Commands: #{ acceptable_commands.sort.join(' ') }"
end

.prompt_choices(choices = []) ⇒ Object

# File 'lib/dataduck/commands.rb', line 20

def self.prompt_choices(choices = [])
  while true
    print "Enter a number 0 - #{ choices.length - 1}\n"
    choices.each_with_index do |choice, idx|
      choice_name = choice.is_a?(String) ? choice : choice[1]
      print "#{ idx }: #{ choice_name }\n"
    end
    choice = STDIN.gets.strip.to_i
    if 0 <= choice && choice < choices.length
      selected = choices[choice]
      return selected.is_a?(String) ? selected : selected[0]
    end
  end
end

.quickstart ⇒ Object

# File 'lib/dataduck/commands.rb', line 177

def self.quickstart
  puts "Welcome to DataDuck!"
  puts "This quickstart wizard will help you set up DataDuck."

  puts "What is your work email address?"
  email = STDIN.gets.strip
  self.quickstart_register_email(email)

  puts "What kind of database would you like to source from?"
  db_type = prompt_choices([
      [:mysql, "MySQL"],
      [:postgresql, "PostgreSQL"],
      [:other, "other"],
  ])

  if db_type == :other
    puts "You've selected 'other'. Unfortunately, those are the only choices supported at the moment. Contact us at DataDuckETL.com to request support for your database."
    exit
  end

  puts "Enter the source hostname:"
  source_host = STDIN.gets.strip

  puts "Enter the name of the database when connecting to #{ source_host }:"
  source_database = STDIN.gets.strip

  puts "Enter the source's port:"
  source_port = STDIN.gets.strip.to_i

  puts "Enter the username:"
  source_username = STDIN.gets.strip

  puts "Enter the password:"
  source_password = STDIN.noecho(&:gets).chomp

  db_class = {
      mysql: DataDuck::MysqlSource,
      postgresql: DataDuck::PostgresqlSource,
  }[db_type]

  db_source = db_class.new("source1", {
      'db_type' => db_type.to_s,
      'host' => source_host,
      'database' => source_database,
      'port' => source_port,
      'username' => source_username,
      'password' => source_password,
  })

  puts "Connecting to source database..."
  table_names = db_source.table_names
  puts "Connection successful. Detected #{ table_names.length } tables."
  puts "Creating scaffolding..."
  table_names.each do |table_name|
    DataDuck::Commands.quickstart_create_table(table_name, db_source)
  end

  config_obj = {
    'users' => {
        email => {
            'admin' => true
        }
    },
    'sources' => {
      'source1' => {
        'type' => db_type.to_s,
        'host' => source_host,
        'database' => source_database,
        'port' => source_port,
        'username' => source_username,
      }
    },
    'destinations' => {
      'destination1' => {
        'type'  => 'redshift',
        's3_bucket'  => 'YOUR_BUCKET',
        's3_region'  => 'YOUR_BUCKET_REGION',
        'host'  => 'redshift.somekeygoeshere.us-west-2.redshift.amazonaws.com',
        'port'  => 5439,
        'database'  => 'main',
        'schema'  => 'public',
        'username'  => 'YOUR_UESRNAME',
      }
    }
  }
  DataDuck::Commands.quickstart_save_file("#{ DataDuck.project_root }/config/base.yml", config_obj.to_yaml)

  DataDuck::Commands.quickstart_save_file("#{ DataDuck.project_root }/.env", """
destination1_aws_key=AWS_KEY_GOES_HERE
destination1_aws_secret=AWS_SECRET_GOES_HERE
destination1_password=REDSHIFT_PASSWORD_GOES_HERE
source1_password=#{ source_password }
""".strip)

  DataDuck::Commands.quickstart_update_gitignore

  puts "Quickstart complete!"
  puts "You still need to edit your .env and config/base.yml files with your AWS and Redshift credentials."
  puts "Run your ETL with: dataduck etl all"
  puts "For more help, visit http://dataducketl.com/docs"
end

.quickstart_create_table(table_name, db) ⇒ Object

# File 'lib/dataduck/commands.rb', line 288

def self.quickstart_create_table(table_name, db)
  columns = []
  schema = db.schema(table_name)
  schema.each do |property_schema|
    property_name = property_schema[0]
    property_type = property_schema[1][:type]
    commented_out = ['ssn', 'socialsecurity', 'password', 'encrypted_password', 'salt', 'password_salt', 'pw'].include?(property_name.to_s.downcase)
    columns << [property_name.to_s, property_type.to_s, commented_out]
  end

  columns.sort! { |a, b| a[0] <=> b[0] }

  table_name = table_name.to_s.downcase
  table_name_camelcased = table_name.split('_').collect(&:capitalize).join
  namespace = Namespace.new(table_name_camelcased: table_name_camelcased, table_name: table_name, columns: columns)
  template = File.open("#{ DataDuck.gem_root }/lib/templates/quickstart/table.rb.erb", 'r').read
  result = ERB.new(template).result(namespace.get_binding)
  DataDuck::Commands.quickstart_save_file("#{ DataDuck.project_root }/src/tables/#{ table_name }.rb", result)
end

.quickstart_register_email(email) ⇒ Object

# File 'lib/dataduck/commands.rb', line 157

def self.quickstart_register_email(email)
  registration_data = {
      email: email,
      version: DataDuck::VERSION,
      source: "quickstart"
  }

  request = Typhoeus::Request.new(
      "dataducketl.com/api/v1/register",
      method: :post,
      body: registration_data,
      timeout: 30,
      connecttimeout: 10,
  )

  hydra = Typhoeus::Hydra.new
  hydra.queue(request)
  hydra.run
end

.quickstart_save_file(output_path_full, contents) ⇒ Object

# File 'lib/dataduck/commands.rb', line 308

def self.quickstart_save_file(output_path_full, contents)
  *output_path, output_filename = output_path_full.split('/')
  output_path = output_path.join("/")
  FileUtils::mkdir_p(output_path)

  output = File.open(output_path_full, "w")
  output << contents
  output.close
end

.quickstart_update_gitignore ⇒ Object

# File 'lib/dataduck/commands.rb', line 279

def self.quickstart_update_gitignore
  main_gitignore_path = "#{ DataDuck.project_root }/.gitignore"
  FileUtils.touch(main_gitignore_path)
  output = File.open(main_gitignore_path, "w")
  output << ".DS_Store\n"
  output << ".env\n"
  output.close
end

.recreate(table_name) ⇒ Object

# File 'lib/dataduck/commands.rb', line 122

def self.recreate(table_name)
  table_name_camelized = DataDuck::Util.underscore_to_camelcase(table_name)
  require DataDuck.project_root + "/src/tables/#{ table_name }.rb"
  table_class = Object.const_get(table_name_camelized)
  if !(table_class <= DataDuck::Table)
    raise Exception.new("Table class #{ table_name_camelized } must inherit from DataDuck::Table")
  end
  table = table_class.new
  table.recreate!(DataDuck::Destination.only_destination)
end

.route_command(args) ⇒ Object

# File 'lib/dataduck/commands.rb', line 39

def self.route_command(args)
  if args.length == 0
    return DataDuck::Commands.help
  end

  command = args[0]
  if !Commands.acceptable_commands.include?(command)
    puts "No such command: #{ command }"
    return DataDuck::Commands.help
  end

  DataDuck::Commands.public_send(command, *args[1..-1])
end

.show(table_name = nil) ⇒ Object

# File 'lib/dataduck/commands.rb', line 133

def self.show(table_name = nil)
  if table_name.nil?
    Dir[DataDuck.project_root + "/src/tables/*.rb"].each do |file|
      table_name_underscores = file.split("/").last.gsub(".rb", "")
      table_name_camelized = DataDuck::Util.underscore_to_camelcase(table_name_underscores)
      require file
      table = Object.const_get(table_name_camelized)
      if table <= DataDuck::Table
        puts table_name_underscores
      end
    end
  else
    table_name_camelized = DataDuck::Util.underscore_to_camelcase(table_name)
    require DataDuck.project_root + "/src/tables/#{ table_name }.rb"
    table_class = Object.const_get(table_name_camelized)
    if !(table_class <= DataDuck::Table)
      raise Exception.new("Table class #{ table_name_camelized } must inherit from DataDuck::Table")
    end

    table = table_class.new
    table.show
  end
end