Class: DataDuck::Commands

Inherits:

Object

• Object
• DataDuck::Commands

Defined in:

lib/dataduck/commands.rb

Defined Under Namespace

Classes: Namespace

Class Method Summary

• .acceptable_commands ⇒ Object
• .c ⇒ Object
• .console ⇒ Object
• .d(where = "destination") ⇒ Object
• .dbconsole(where = "destination") ⇒ Object
• .etl(*table_names_underscore) ⇒ Object
• .help ⇒ Object
• .prompt_choices(choices = []) ⇒ Object
• .quickstart ⇒ Object
• .quickstart_create_schedule_config ⇒ Object
• .quickstart_create_table(table_name, db) ⇒ Object
• .quickstart_register_email(email) ⇒ Object
• .quickstart_save_file(output_path_full, contents) ⇒ Object
• .quickstart_update_gitignore ⇒ Object
• .recreate(table_name) ⇒ Object
• .route_command(args) ⇒ Object
• .show(table_name = nil) ⇒ Object

Class Method Details

.acceptable_commands ⇒ Object

# File 'lib/dataduck/commands.rb', line 36

def self.acceptable_commands
  ['c', 'console', 'd', 'dbconsole', 'etl', 'quickstart', 'recreate', 'show']
end

.c ⇒ Object

# File 'lib/dataduck/commands.rb', line 58

def self.c
  self.console
end

.console ⇒ Object

# File 'lib/dataduck/commands.rb', line 62

def self.console
  require "irb"
  ARGV.clear
  IRB.start
end

.d(where = "destination") ⇒ Object

# File 'lib/dataduck/commands.rb', line 68

def self.d(where = "destination")
  self.dbconsole(where)
end

.dbconsole(where = "destination") ⇒ Object

# File 'lib/dataduck/commands.rb', line 72

def self.dbconsole(where = "destination")
  which_database = nil
  if where == "destination"
    which_database = DataDuck::Destination.only_destination
  elsif where == "source"
    which_database = DataDuck::Source.only_source
  else
    found_source = DataDuck::Source.source(where, true)
    found_destination = DataDuck::Destination.destination(where, true)
    if found_source && found_destination
      raise ArgumentError.new("Ambiguous call to dbconsole for #{ where } since there is both a source and destination named #{ where }.")
    end

    which_database = found_source if found_source
    which_database = found_destination if found_destination
  end

  if which_database.nil?
    raise ArgumentError.new("Could not find database '#{ where }'")
  end

  puts "Connecting to #{ where }..."
  which_database.dbconsole
end

.etl(*table_names_underscore) ⇒ Object

# File 'lib/dataduck/commands.rb', line 97

def self.etl(*table_names_underscore)
  if table_names_underscore.length == 0
    puts "You need to specify a table name or 'all'. Usage: dataduck etl all OR datduck etl my_table_name"
    return
  end

  only_destination = DataDuck::Destination.only_destination

  etl = nil
  if table_names_underscore.length == 1 && table_names_underscore[0] == "all"
    etl = ETL.new(destinations: [only_destination], autoload_tables: true)
  else
    tables = []
    table_names_underscore.each do |table_name|
      table_name_camelized = DataDuck::Util.underscore_to_camelcase(table_name)
      require DataDuck.project_root + "/src/tables/#{ table_name }.rb"
      table_class = Object.const_get(table_name_camelized)
      if !(table_class <= DataDuck::Table)
        raise "Table class #{ table_name_camelized } must inherit from DataDuck::Table"
      end
      table = table_class.new
      tables << table
    end
    etl = ETL.new({
        destinations: [only_destination],
        autoload_tables: false,
        tables: tables
    })
  end
  etl.process!
  
  if etl.errored?
    exit(1)
  end
end

.help ⇒ Object

# File 'lib/dataduck/commands.rb', line 133

def self.help
  puts "Usage: dataduck commandname"
  puts "Commands: #{ acceptable_commands.sort.join(' ') }"
end

.prompt_choices(choices = []) ⇒ Object

# File 'lib/dataduck/commands.rb', line 21

def self.prompt_choices(choices = [])
  while true
    print "Enter a number 0 - #{ choices.length - 1}\n"
    choices.each_with_index do |choice, idx|
      choice_name = choice.is_a?(String) ? choice : choice[1]
      print "#{ idx }: #{ choice_name }\n"
    end
    choice = STDIN.gets.strip.to_i
    if 0 <= choice && choice < choices.length
      selected = choices[choice]
      return selected.is_a?(String) ? selected : selected[0]
    end
  end
end

.quickstart ⇒ Object

# File 'lib/dataduck/commands.rb', line 193

def self.quickstart
  puts "Welcome to DataDuck!"
  puts "This quickstart wizard will help you set up DataDuck."

  puts "What is your work email address?"
  email = STDIN.gets.strip
  self.quickstart_register_email(email)

  puts "What kind of database would you like to source from?"
  db_type = prompt_choices([
      [:mysql, "MySQL"],
      [:postgresql, "PostgreSQL"],
      [:other, "other"],
  ])

  if db_type == :other
    puts "You've selected 'other'. Unfortunately, those are the only choices supported at the moment. Contact us at DataDuckETL.com to request support for your database."
    exit
  end

  puts "Enter the source hostname:"
  source_host = STDIN.gets.strip

  puts "Enter the name of the database when connecting to #{ source_host }:"
  source_database = STDIN.gets.strip

  puts "Enter the source's port:"
  source_port = STDIN.gets.strip.to_i

  puts "Enter the username:"
  source_username = STDIN.gets.strip

  puts "Enter the password:"
  source_password = STDIN.noecho(&:gets).chomp

  db_class = {
      mysql: DataDuck::MysqlSource,
      postgresql: DataDuck::PostgresqlSource,
  }[db_type]

  db_source = db_class.new("source1", {
      'db_type' => db_type.to_s,
      'host' => source_host,
      'database' => source_database,
      'port' => source_port,
      'username' => source_username,
      'password' => source_password,
  })

  puts "Connecting to source database..."
  table_names = db_source.table_names
  puts "Connection successful. Detected #{ table_names.length } tables."
  puts "Creating scaffolding..."
  table_names.each do |table_name|
    begin
      DataDuck::Commands.quickstart_create_table(table_name, db_source)
    rescue
    end
  end

  config_obj = {
    'users' => {
        email => {
            'admin' => true
        }
    },
    'sources' => {
      'source1' => {
        'type' => db_type.to_s,
        'host' => source_host,
        'database' => source_database,
        'port' => source_port,
        'username' => source_username,
      }
    },
    'destinations' => {
      'destination1' => {
        'type'  => 'redshift',
        's3_bucket'  => 'YOUR_BUCKET',
        's3_region'  => 'YOUR_BUCKET_REGION',
        'host'  => 'redshift.somekeygoeshere.us-west-2.redshift.amazonaws.com',
        'port'  => 5439,
        'database'  => 'main',
        'schema'  => 'public',
        'username'  => 'YOUR_UESRNAME',
      }
    }
  }
  DataDuck::Commands.quickstart_save_file("#{ DataDuck.project_root }/config/base.yml", config_obj.to_yaml)

  DataDuck::Commands.quickstart_save_file("#{ DataDuck.project_root }/.env", """
destination1_aws_key=AWS_KEY_GOES_HERE
destination1_aws_secret=AWS_SECRET_GOES_HERE
destination1_password=REDSHIFT_PASSWORD_GOES_HERE
source1_password=#{ source_password }
""".strip)

  DataDuck::Commands.quickstart_update_gitignore

  puts "Quickstart complete!"
  puts "You still need to edit your .env and config/base.yml files with your AWS and Redshift credentials."
  puts "Run your ETL with: dataduck etl all"
  puts "For more help, visit http://dataducketl.com/docs"
end

.quickstart_create_schedule_config ⇒ Object

# File 'lib/dataduck/commands.rb', line 327

def self.quickstart_create_schedule_config
  namespace = Namespace.new
  template = File.open("#{ DataDuck.gem_root }/lib/templates/quickstart/schedule.rb.erb", 'r').read
  result = ERB.new(template).result(namespace.get_binding)
  DataDuck::Commands.quickstart_save_file("#{ DataDuck.project_root }/config/schedule.rb", result)
end

.quickstart_create_table(table_name, db) ⇒ Object

# File 'lib/dataduck/commands.rb', line 307

def self.quickstart_create_table(table_name, db)
  columns = []
  schema = db.schema(table_name)
  schema.each do |property_schema|
    property_name = property_schema[0]
    property_type = property_schema[1][:type]
    commented_out = ['ssn', 'socialsecurity', 'password', 'encrypted_password', 'salt', 'password_salt', 'pw'].include?(property_name.to_s.downcase)
    columns << [property_name.to_s, property_type.to_s, commented_out]
  end

  columns.sort! { |a, b| a[0] <=> b[0] }

  table_name = table_name.to_s.downcase
  table_name_camelcased = table_name.split('_').collect(&:capitalize).join
  namespace = Namespace.new(table_name_camelcased: table_name_camelcased, table_name: table_name, columns: columns)
  template = File.open("#{ DataDuck.gem_root }/lib/templates/quickstart/table.rb.erb", 'r').read
  result = ERB.new(template).result(namespace.get_binding)
  DataDuck::Commands.quickstart_save_file("#{ DataDuck.project_root }/src/tables/#{ table_name }.rb", result)
end

.quickstart_register_email(email) ⇒ Object

# File 'lib/dataduck/commands.rb', line 173

def self.quickstart_register_email(email)
  registration_data = {
      email: email,
      version: DataDuck::VERSION,
      source: "quickstart"
  }

  request = Typhoeus::Request.new(
      "dataducketl.com/api/v1/register",
      method: :post,
      body: registration_data,
      timeout: 30,
      connecttimeout: 10,
  )

  hydra = Typhoeus::Hydra.new
  hydra.queue(request)
  hydra.run
end

.quickstart_save_file(output_path_full, contents) ⇒ Object

# File 'lib/dataduck/commands.rb', line 334

def self.quickstart_save_file(output_path_full, contents)
  *output_path, output_filename = output_path_full.split('/')
  output_path = output_path.join("/")
  FileUtils::mkdir_p(output_path)

  output = File.open(output_path_full, "w")
  output << contents
  output.close
end

.quickstart_update_gitignore ⇒ Object

# File 'lib/dataduck/commands.rb', line 298

def self.quickstart_update_gitignore
  main_gitignore_path = "#{ DataDuck.project_root }/.gitignore"
  FileUtils.touch(main_gitignore_path)
  output = File.open(main_gitignore_path, "w")
  output << ".DS_Store\n"
  output << ".env\n"
  output.close
end

.recreate(table_name) ⇒ Object

# File 'lib/dataduck/commands.rb', line 138

def self.recreate(table_name)
  table_name_camelized = DataDuck::Util.underscore_to_camelcase(table_name)
  require DataDuck.project_root + "/src/tables/#{ table_name }.rb"
  table_class = Object.const_get(table_name_camelized)
  if !(table_class <= DataDuck::Table)
    raise "Table class #{ table_name_camelized } must inherit from DataDuck::Table"
  end
  table = table_class.new
  table.recreate!(DataDuck::Destination.only_destination)
end

.route_command(args) ⇒ Object

# File 'lib/dataduck/commands.rb', line 40

def self.route_command(args)
  if args.length == 0
    return DataDuck::Commands.help
  end

  command = args[0]
  if !Commands.acceptable_commands.include?(command)
    puts "No such command: #{ command }"
    return DataDuck::Commands.help
  end

  begin
    DataDuck::Commands.public_send(command, *args[1..-1])
  rescue => err
    DataDuck::Logs.error(err)
  end
end

.show(table_name = nil) ⇒ Object

# File 'lib/dataduck/commands.rb', line 149

def self.show(table_name = nil)
  if table_name.nil?
    Dir[DataDuck.project_root + "/src/tables/*.rb"].each do |file|
      table_name_underscores = file.split("/").last.gsub(".rb", "")
      table_name_camelized = DataDuck::Util.underscore_to_camelcase(table_name_underscores)
      require file
      table = Object.const_get(table_name_camelized)
      if table <= DataDuck::Table
        puts table_name_underscores
      end
    end
  else
    table_name_camelized = DataDuck::Util.underscore_to_camelcase(table_name)
    require DataDuck.project_root + "/src/tables/#{ table_name }.rb"
    table_class = Object.const_get(table_name_camelized)
    if !(table_class <= DataDuck::Table)
      raise "Table class #{ table_name_camelized } must inherit from DataDuck::Table"
    end

    table = table_class.new
    table.show
  end
end