Class: PoolParty::Plugin::Hive

Inherits:
PoolParty::Plugin show all
Defined in:
lib/extensions/hive/hive.rb

Instance Method Summary collapse

Instance Method Details

#before_load(o = {}, &block) ⇒ Object 



8
9
10
11
12
13
14
15
 
# File 'lib/extensions/hive/hive.rb', line 8

def before_load(o={}, &block)
  do_once do
    # install_from_bin
    install_from_src
    set_environment_variables
    create_hdfs_directories
  end
end

#create_hdfs_directories ⇒ Object 



67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
 
# File 'lib/extensions/hive/hive.rb', line 67

def create_hdfs_directories
  has_exec "#{hadoop_home}/bin/hadoop fs -mkdir /tmp", 
    :not_if => "#{hadoop_home}/bin/hadoop fs -ls /tmp",
    :only_if => "test -e #{hadoop_data_dir}/dfs && (ps aux | grep org.apache.hadoop.hdfs.server.namenode.NameNode | grep -v grep)"

  has_exec "#{hadoop_home}/bin/hadoop fs -mkdir /user/hive/warehouse", 
    :not_if => "#{hadoop_home}/bin/hadoop fs -ls /user/hive/warehouse",
    :only_if => "test -e #{hadoop_data_dir}/dfs && (ps aux | grep org.apache.hadoop.hdfs.server.namenode.NameNode | grep -v grep)"

  has_exec "#{hadoop_home}/bin/hadoop fs -chmod g+w /tmp", 
    :not_if => "#{hadoop_home}/bin/hadoop fs -ls /tmp", # todo, check perms
    :only_if => "test -e #{hadoop_data_dir}/dfs && (ps aux | grep org.apache.hadoop.hdfs.server.namenode.NameNode | grep -v grep)"
 
  has_exec "#{hadoop_home}/bin/hadoop fs -chmod g+w /user/hive/warehouse", 
    :not_if => "#{hadoop_home}/bin/hadoop fs -ls /user/hive/warehouse",
    :only_if => "test -e #{hadoop_data_dir}/dfs && (ps aux | grep org.apache.hadoop.hdfs.server.namenode.NameNode | grep -v grep)"
end

#download_and_build_src ⇒ Object 



43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/extensions/hive/hive.rb', line 43

def download_and_build_src
  has_exec "svn co #{hive_repo} #{src_dir} -r#{hive_revision}",
    :not_if => "test -e #{src_dir}/build.xml"
  has_exec "cd #{src_dir} && wget --no-check-certificate https://issues.apache.org/jira/secure/attachment/12409779/hive-487.3.patch",
    :not_if => "test -e #{src_dir}/hive-487.3.patch"
  has_exec "cd #{src_dir} && patch -p0 < hive-487.3.patch && mv hive-487.3.patch hive-487.3.patch.applied", 
    :not_if => "test -e #{src_dir}/hive-487.3.patch.applied"
  has_exec "cd #{src_dir} && ant -Dhadoop.version=\\\"#{hadoop_version}\\\" package",
    :not_if => "test -e #{hive_home}/README.txt"
  has_exec "mv #{src_dir}/build/dist #{hive_home}",
    :not_if => "test -e #{hive_home}"
end

#install_dependent_packages ⇒ Object 



38
39
40
41
 
# File 'lib/extensions/hive/hive.rb', line 38

def install_dependent_packages
  has_package :name => "subversion"
  has_package :name => "ant"
end

#install_from_bin ⇒ Object 



23
24
25
26
27
28
29
30
 
# File 'lib/extensions/hive/hive.rb', line 23

def install_from_bin
  has_exec "wget #{hive_dist} -O /usr/local/src/hive-0.3.0-hadoop-0.19.0-dev.tar.gz",
    :not_if => "test -e /usr/local/src/hive-0.3.0-hadoop-0.19.0-dev.tar.gz"
  has_exec "cd /usr/local/src && tar -xvvf /usr/local/src/hive-0.3.0-hadoop-0.19.0-dev.tar.gz", 
    :not_if => "test -e #{hive_home}"
  has_exec "mv /usr/local/src/hive-0.3.0-hadoop-0.19.0-dev #{hive_home}", 
    :not_if => "test -e #{hive_home}"
end

#install_from_src ⇒ Object

doesn’t really work



33
34
35
36
 
# File 'lib/extensions/hive/hive.rb', line 33

def install_from_src
  install_dependent_packages
  download_and_build_src
end

#loaded(o = {}, &block) ⇒ Object 



17
18
19
20
21
 
# File 'lib/extensions/hive/hive.rb', line 17

def loaded(o={}, &block)
  do_once do
    has_file :name => "#{hive_home}/conf/hive-default.xml", :template => "hive-default.xml.erb"
  end
end

#set_environment_variables ⇒ Object

todo, pull from parent



57
58
59
60
61
62
63
64
65
 
# File 'lib/extensions/hive/hive.rb', line 57

def set_environment_variables
  has_file :name => "/root/.hadoop-etc-env.sh", :content => <<-EOF
export HADOOP_HOME=#{hadoop_home}
export HADOOP=$HADOOP_HOME/bin/hadoop
export HIVE_HOME=#{hive_home}
export PATH=$HADOOP_HOME/bin:$HIVE_HOME/bin:$PATH
  EOF
  has_line_in_file :file => "/root/.profile", :line => "source /root/.hadoop-etc-env.sh"
end

#use_mysql(host, user, pass) ⇒ Object

you can also pass ‘host:port’ for host



86
87
88
89
90
91
92
93
94
95
96
97
 
# File 'lib/extensions/hive/hive.rb', line 86

def use_mysql(host, user, pass)
  has_exec "wget http://mysql.mirrors.pair.com/Downloads/Connector-J/mysql-connector-java-5.0.8.tar.gz -O /usr/local/src/mysql-connector-java-5.0.8.tar.gz", 
    :not_if => "test -e /usr/local/src/mysql-connector-java-5.0.8.tar.gz"
  has_exec "cd /usr/local/src && tar -xvvf /usr/local/src/mysql-connector-java-5.0.8.tar.gz", 
    :not_if => "test -e /usr/local/src/mysql-connector-java-5.0.8"
  has_exec "mv /usr/local/src/mysql-connector-java-5.0.8/mysql-connector-java-5.0.8-bin.jar #{hive_home}/lib", 
    :not_if => "test -e #{hive_home}/lib/mysql-connector-java-5.0.8-bin.jar"
  has_variable :name => "hive_mysql_enabled", :value => "true"
  has_variable :name => "hive_mysql_host", :value => host
  has_variable :name => "hive_mysql_user", :value => user
  has_variable :name => "hive_mysql_pass", :value => pass
end