Class: SparkToolkit::Conf::Configuration

Inherits:
Object
  • Object
show all
Defined in:
lib/spark_toolkit/hadoop/conf/configuration.rb

Instance Method Summary collapse

Constructor Details

#initialize(opts = {}) ⇒ Configuration

Returns a new instance of Configuration.



9
10
11
12
13
14
15
16
17
18
 
# File 'lib/spark_toolkit/hadoop/conf/configuration.rb', line 9

def initialize(opts={})
  initialise

  default_opts = {
    'fs.hdfs.impl' => 'org.apache.hadoop.hdfs.DistributedFileSystem',
    'fs.file.impl' => 'org.apache.hadoop.fs.LocalFileSystem'
  }

  default_opts.merge(opts).each { |k, v| set(k, v) }
end

Instance Method Details

#[](k) ⇒ Object 



35
36
37
 
# File 'lib/spark_toolkit/hadoop/conf/configuration.rb', line 35

def [](k)
  get(k)
end

#[]=(k, v) ⇒ Object 



31
32
33
 
# File 'lib/spark_toolkit/hadoop/conf/configuration.rb', line 31

def []=(k, v)
  set(k, v)
end

#add_config_dir(dir) ⇒ Object

Load *.xml files under input dir



40
41
42
 
# File 'lib/spark_toolkit/hadoop/conf/configuration.rb', line 40

def add_config_dir(dir)
  Dir.glob("#{dir}/*.xml").each { |f| add_resource(f) }
end

#add_resource(f) ⇒ Object 



21
22
23
 
# File 'lib/spark_toolkit/hadoop/conf/configuration.rb', line 21

def add_resource(f)
  add_resource_java(Path.new(f))
end

#add_resource_java ⇒ Object 



20
 
# File 'lib/spark_toolkit/hadoop/conf/configuration.rb', line 20

alias_method :add_resource_java, :add_resource

#initialise ⇒ Object 



8
 
# File 'lib/spark_toolkit/hadoop/conf/configuration.rb', line 8

alias_method :initialise, :initialize

#krb_login(principle, keytab) ⇒ Object 



25
26
27
28
29
 
# File 'lib/spark_toolkit/hadoop/conf/configuration.rb', line 25

def (principle, keytab)
  set('hadoop.security.authentication', 'kerberos')
  UserGroupInformation.set_configuration(self)
  UserGroupInformation.(principle, keytab)
end