用shell脚本自动化安装hadoop

 1 # Shell Script
 2 export SHELL_LOC=`pwd`
 3 export CONF_LOC=$SHELL_LOC/conf
 4 export TOOLS_LOC=$SHELL_LOC/tools
 5 
 6 # Hosts
 7 export HOSTS=`cat $CONF_LOC/masters $CONF_LOC/slaves`
 8 export MASTER=`cat $CONF_LOC/masters`
 9 export SLAVES=`cat $CONF_LOC/slaves`
10 export IP_LISTS=`cat $CONF_LOC/masters $CONF_LOC/slaves`
11 export IP_COUNT=`cat $CONF_LOC/masters $CONF_LOC/slaves | wc -l`
12 
13 # Users
14 export ROOT_USERPWD=Passw0rd
15 export ROOT_PROMPT=]#
16 export HADOOP_USERNAME=hadoop
17 export HADOOP_USERPWD=hadoop
18 export HADOOP_PROMPT=]$
19 export HADOOP_USERHOME=/home/$HADOOP_USERNAME
20 
21 # Expect Command
22 export EXPECTCHK=`rpm -qa expect | wc -l`
23 export EXPECT_TIMEOUT=4
24 export EXPECT_TIMEOUT_BIG=600
25 
26 # Java
27 export JAVA_PKGNAME=jdk-7u51-linux-x64.tar.gz
28 export JAVA_FOLDERNAME=jdk1.7.0_51    # 与实际 java 压缩包解压后的文件夹名称相同
29 export JAVA_INSTLOC=/usr/java
30 export JAVA_HOME=$JAVA_INSTLOC/$JAVA_FOLDERNAME
31 
32 # Hadoop
33 export HADOOP_PKGNAME=hadoop-1.2.1.tar.gz
34 export HADOOP_FOLDERNAME=hadoop-1.2.1    # 与实际 hadoop 压缩包解压后的文件夹名称相同
35 export HADOOP_INSTLOC=/usr    # 如果修改此值，请一并修改 conf/hadoop.env 文件中内容
36 export HADOOP_HOME=$HADOOP_INSTLOC/$HADOOP_FOLDERNAME
37 export HADOOP_CONFLOC=$HADOOP_HOME/conf
38 export HADOOP_TMP_DIR=$HADOOP_HOME/tmp
39 export HADOOP_DFS_REPLICATION=1

1 #set java environment
2 export JAVA_HOME=/usr/java/jdk1.7.0_51
3 export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
4 export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH

1 #set hadoop path
2 export HADOOP_HOME=/usr/hadoop-1.2.1
3 export PATH=$PATH:$HADOOP_HOME/bin
4 export HADOOP_HOME_WARN_SUPPRESS=1

 1     <!-- temp folder properties -->
 2     <property>
 3         <name>hadoop.tmp.dir</name>
 4         <value>/usr/hadoop-1.2.1/tmp</value>
 5         <description>A base for other temporary directories.</description>
 6     </property>
 7     <!-- file system properties -->
 8     <property>
 9         <name>fs.default.name</name>
10         <value>hdfs://192.168.1.144:9000</value>
11     </property>
12 </configuration>

1     <property>
2         <name>dfs.replication</name>
3         <value>1</value>
4     </property>
5 </configuration>

1     <property>
2         <name>mapred.job.tracker</name>
3         <value>http://192.168.1.144:9001</value>
4     </property>
5 </configuration>

 1 #!/bin/bash
 2 
 3 # Purpose:    Install hadoop env on clean hosts.
 4 # Author:       13Bear
 5 # Update:       2014-04-25 17:18
 6 
 7 # Exit value:
 8 # 0     Install succeed
 9 # 1     Some hosts can‘t be reached
10 
11 # Import env variable
12 source ./conf/env.config
13 
14 # Start install
15 start_time=`date`
16 
17 # Make sure all the hosts can be reach
18 ./pingAll.sh
19 if [ $? -eq 1 ]; then
20     echo "###############################################"
21     echo -e "\033[31mSome nodes can‘t be reached! Please check them!\033[0m"
22     echo "###############################################"
23     exit 1
24 fi
25 # Start install
26 echo ""
27 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
28 echo "++++++++++++++++++++++++++++++++++++++++++++++++ Start Install +++++++++++++++++++++++++++++++++++++++++++++++"
29 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
30 echo ""
31 ./useraddAll.sh
32 ./sshAll.sh
33 ./javaAll.sh
34 ./hostsAll.sh
35 ./hadoopAll.sh
36 echo ""
37 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
38 echo "+++++++++++++++++++++++++++++++++++++++++++++++++ Eed Install ++++++++++++++++++++++++++++++++++++++++++++++++"
39 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
40 echo ""
41 
42 # 统计安装的开始、结束时间。
43 end_time=`date`
44 echo "###############################################"
45 echo "Start Time: $start_time"
46 echo "End Time:   $end_time"
47 echo "###############################################"
48 exit 0

 1 #!/bin/bash
 2 
 3 # Purpose:    Test whether the ips all can be pinged.
 4 # Author:       13Bear
 5 # Update:       2014-04-25 17:18
 6 
 7 # Exit value:
 8 # 0     All succeed
 9 # 1     Some failed
10 
11 # Import env variable
12 source ./conf/env.config
13 
14 echo ""
15 echo "+++++++++++++++++++++++++++++++++++++++++++++++ Ping Hosts Process +++++++++++++++++++++++++++++++++++++++++++++++"
16 echo ""
17 
18 # Some variable
19 SUCCEED_COUNT=0
20 FAILED_COUNT=0
21 
22 # Ping all the ips
23 echo "###############################################"
24 echo "Pinging all the hosts of hadoop environment."
25 echo "###############################################"
26 for ip in $IP_LISTS
27 do
28     ping -c 1 $ip  >/dev/null 2>&1
29     # Check the result
30     if [ $? -eq 0 ]; then
31         echo "$ip CAN be reached."
32         let SUCCEED_COUNT+=1
33     else
34         echo -e "\033[31m$ip CANNOT be reached.\033[0m"
35         let FAILED_COUNT+=1
36     fi
37 done
38 
39 # Result
40 if [ $SUCCEED_COUNT -eq $IP_COUNT ]; then
41     #echo "All succeed!"
42     exit 0
43 else
44     #echo "$FAILED_COUNT failed!"
45     exit 1
46 fi

 1 #!/bin/bash
 2 
 3 # Purpose:    Config the hadoop env‘s ssh no password login
 4 # Author:       13Bear
 5 # Update:       2014-04-25 17:18
 6 
 7 # Exit value:
 8 # 0     All succeed
 9 # 1     Some failed
10 # 98    expect not install
11 # 99    Usage format error
12 
13 # Import env variable
14 source ./conf/env.config
15 
16 # Start
17 echo ""
18 echo "+++++++++++++++++++++++++++++++++++++++++++++++ Useradd Process +++++++++++++++++++++++++++++++++++++++++++++++" 
19 echo ""
20 
21 # Check the expect tool‘s installation
22 if [ $EXPECTCHK != 1 ]; then
23         echo "###############################################"
24         echo "Please install the \"expect\" package first on all nodes to allow the script to run"
25         echo "yum -y install expect"
26         echo "###############################################"
27         exit 98
28 fi
29 
30 # Add hadoop user for every host
31 for host in $HOSTS
32 do
33     echo "###############################################"
34     echo "Adding hadoop user \"$HADOOP_USERNAME\" for host $host"
35     echo "###############################################"
36     expect -c "
37         set timeout $EXPECT_TIMEOUT
38         spawn ssh root@$host
39         expect \"yes/no\" {
40             send \"yes\r\"
41             expect \"password:\"
42             send -- \"$ROOT_USERPWD\r\"
43         } \"password:\" {
44             send -- \"$ROOT_USERPWD\r\"
45         }
46         expect \"$ROOT_PROMPT\"
47             send -- \"userdel -r $HADOOP_USERNAME; useradd -d $HADOOP_USERHOME $HADOOP_USERNAME\r\"
48         expect \"$ROOT_PROMPT\"
49             send \"passwd $HADOOP_USERNAME\r\"
50         expect \"New passwd:\"
51             send -- \"$HADOOP_USERPWD\r\"
52         expect \"Retype new password:\"
53             send -- \"$HADOOP_USERPWD\r\"
54         expect \"$ROOT_PROMPT\"
55     "
56     echo ""
57 done

 1 #!/bin/bash
 2 
 3 # Purpose:    Config the hadoop env‘s ssh no password login
 4 # Author:       13Bear
 5 # Update:       2014-04-25 17:18
 6 
 7 # Exit value:
 8 # 0     All succeed
 9 # 1     Some failed
10 # 98    expect not install
11 # 99    Usage format error
12 
13 # Import env variable
14 source ./conf/env.config
15 
16 # Start
17 echo ""
18 echo "+++++++++++++++++++++++++++++++++++++++++++++++ SSH Configuration Process +++++++++++++++++++++++++++++++++++++++++++++++"
19 echo ""
20 
21 # Check the expect tool‘s installation
22 if [ $EXPECTCHK != 1 ]; then
23         echo "###############################################"
24         echo "Please install the \"expect\" package first on all nodes to allow the script to run"
25         echo "yum -y install expect"
26         echo "###############################################"
27         exit 98
28 fi
29 
30 # Generate RSA keys for every host
31 for host in $HOSTS
32 do
33     echo "###############################################"
34     echo "Generating RSA keys on host $host"
35     echo "###############################################"
36     expect -c "
37         set timeout $EXPECT_TIMEOUT
38         spawn ssh $HADOOP_USERNAME@$host
39         expect \"yes/no\" {
40             send \"yes\r\"
41             expect \"password:\"
42             send -- \"$HADOOP_USERPWD\r\"
43         } \"password:\" {
44             send -- \"$HADOOP_USERPWD\r\"
45         }
46         expect \"$HADOOP_PROMPT\"
47             send \"rm -rf ~/.ssh && ssh-keygen -t rsa -P ‘‘\r\"
48         expect \".ssh/id_rsa\"
49             send \"\r\"
50         expect \"$HADOOP_PROMPT\"
51     "
52     echo ""
53 done
54 
55 # Copy every host‘s RSA pub key to ALL hosts
56 for host in $HOSTS
57 do
58     echo "###############################################"
59     echo "Copying $host RSA pub key to ALL hosts"
60     echo "###############################################"
61     for loop in $HOSTS
62     do
63         echo "==========> Copying $host RSA pub key to $loop"
64         expect -c "
65             set timeout $EXPECT_TIMEOUT
66             spawn ssh $HADOOP_USERNAME@$host
67             expect \"yes/no\" {
68                 send \"yes\r\"
69                 expect \"password:\"
70                 send -- \"$HADOOP_USERPWD\r\"
71             } \"password:\" {
72                 send -- \"$HADOOP_USERPWD\r\"
73             }
74             expect \"$HADOOP_PROMPT\"
75             send \"ssh-copy-id -i ~/.ssh/id_rsa.pub $HADOOP_USERNAME@$loop\r\"
76             expect \"yes/no\" {
77                 send \"yes\r\"
78                 expect \"password:\"
79                 send -- \"$HADOOP_USERPWD\r\"
80             } \"password:\" {
81                 send -- \"$HADOOP_USERPWD\r\"
82             }
83             expect \"$HADOOP_PROMPT\"
84         "
85         echo ""
86     done
87 done

 1 #!/bin/bash
 2 
 3 # PurPose:    Config java env
 4 # Author:    13Bear
 5 # Update:    2014-04-25 17:18
 6 
 7 # Exit value:
 8 # 0    All succeed
 9 # 1    Some failed
10 # 98    expect not install
11 # 99    Usage format error
12 
13 # Import env variable
14 source ./conf/env.config
15 
16 # Prepare works
17 # java.env update with variables in env.config
18 sed -i "s#JAVA_HOME=.*#JAVA_HOME=$JAVA_HOME#" $CONF_LOC/java.env
19 
20 # Start
21 echo ""
22 echo "+++++++++++++++++++++++++++++++++++++++++++++++ Java Configuration Process +++++++++++++++++++++++++++++++++++++++++++++++"
23 echo ""
24 
25 # Check the expect tool‘s installation
26 if [ $EXPECTCHK != 1 ]; then
27     echo "###############################################"
28     echo "Please install the \"expect\" package first on all nodes to allow the script to run"
29     echo "yum -y install expect"
30     echo "###############################################"
31     exit 98
32 fi
33 
34 # Configure Java env for all hosts
35 for host in $HOSTS
36 do
37     echo "###############################################"
38     echo "Configuring Java env on $host"
39     echo "###############################################"
40     echo "==========> 1.Copying Java package to $host"
41     expect -c "
42         set timeout $EXPECT_TIMEOUT_BIG
43         spawn scp $TOOLS_LOC/$JAVA_PKGNAME root@$host:/usr
44         expect \"yes/no\" {
45             send \"yes\r\"
46             expect \"password:\"
47             send -- \"$ROOT_USERPWD\r\"
48         } \"password:\" {
49             send -- \"$ROOT_USERPWD\r\"
50         }
51         expect \"$ROOT_PROMPT\"
52     "
53     echo "==========> 2.Extracting Java file to $JAVA_INSTLOC on $host"
54     expect -c "
55         set timeout $EXPECT_TIMEOUT_BIG
56         spawn ssh root@$host
57         expect \"password:\"
58             send -- \"$ROOT_USERPWD\r\"
59         expect \"$ROOT_PROMPT\"
60             send -- \"rm -rf $JAVA_HOME \r\"
61         expect \"$ROOT_PROMPT\"
62             send -- \"mkdir $JAVA_INSTLOC 2>/dev/null \r\"
63         expect \"$ROOT_PROMPT\"
64             send -- \"tar -xzf /usr/$JAVA_PKGNAME -C $JAVA_INSTLOC \r\"
65         expect \"$ROOT_PROMPT\"
66             send -- \"chown -R root:root $JAVA_INSTLOC/$JAVA_FOLDERNAME \r\"
67         expect \"$ROOT_PROMPT\"
68             send -- \"rm -f /usr/$JAVA_PKGNAME \r\"
69         expect \"$ROOT_PROMPT\"
70     "
71     echo ""
72     echo "==========> 3.Configuring Java env varibales on $host"
73     expect -c "
74         set timeout $EXPECT_TIMEOUT_BIG
75         spawn scp $CONF_LOC/java.env root@$host:~/
76         expect \"password:\"
77         send -- \"$ROOT_USERPWD\r\"
78         expect \"$ROOT_PROMPT\"
79     "
80     expect -c "
81         set timeout $EXPECT_TIMEOUT_BIG
82         spawn ssh root@$host
83         expect \"password:\"
84             send -- \"$ROOT_USERPWD\r\"
85         expect \"$ROOT_PROMPT\"
86             send -- \"sed -i ‘/java/Id‘ /etc/profile \r\"
87         expect \"$ROOT_PROMPT\"
88             send -- \"cat ~/java.env >> /etc/profile \r\"
89         expect \"$ROOT_PROMPT\"
90             send -- \"rm -f ~/java.env \r\"
91         expect \"$ROOT_PROMPT\"
92             send -- \"source /etc/profile \r\"
93         expect \"$ROOT_PROMPT\"
94             send -- \"java -version \r\"
95         expect \"$ROOT_PROMPT\"
96     "
97     echo ""
98 done

 1 #!/bin/bash
 2 
 3 # PurPose:    Add hadoop hosts to /etc/hosts
 4 # Author:    13Bear
 5 # Update:    2014-04-25 17:18
 6 
 7 # Exit value:
 8 # 0    All succeed
 9 # 1    Some failed
10 # 98    expect not install
11 # 99    Usage format error
12 
13 # Import env variable
14 source ./conf/env.config
15 
16 # Start
17 echo ""
18 echo "+++++++++++++++++++++++++++++++++++++++++++++++ Add hadoop hosts to /etc/hosts Process +++++++++++++++++++++++++++++++++++++++++++++++"
19 echo ""
20 
21 # Check the expect tool‘s installation
22 if [ $EXPECTCHK != 1 ]; then
23     echo "###############################################"
24     echo "Please install the \"expect\" package first on all nodes to allow the script to run"
25     echo "yum -y install expect"
26     echo "###############################################"
27     exit 98
28 fi
29 
30 # Add hadoop hosts to /etc/hosts for all hosts
31 for host in $HOSTS
32 do
33     echo "###############################################"
34     echo "Adding hadoop hosts to /etc/hosts on $host"
35     echo "###############################################"
36     echo "==========> 1.Copying hosts file to $host"
37     expect -c "
38         set timeout $EXPECT_TIMEOUT_BIG
39         spawn scp $CONF_LOC/hosts root@$host:~/
40         expect \"yes/no\" {
41             send \"yes\r\"
42             expect \"password:\"
43             send -- \"$ROOT_USERPWD\r\"
44         } \"password:\" {
45             send -- \"$ROOT_USERPWD\r\"
46         }
47         expect \"$ROOT_PROMPT\"
48     "
49     echo "==========> 2.Adding hosts file content to /etc/hosts on $host"
50     # sed 命令查找删除的关键字 hadoop 一定要在 hosts 文件中包含，不然 sed 删除失败
51     expect -c "
52         set timeout $EXPECT_TIMEOUT_BIG
53         spawn ssh root@$host
54         expect \"password:\"
55             send -- \"$ROOT_USERPWD\r\"
56         expect \"$ROOT_PROMPT\"
57             send -- \"sed -i ‘/hadoop/d‘ /etc/hosts \r\"
58         expect \"$ROOT_PROMPT\"
59             send -- \"cat ~/hosts >> /etc/hosts \r\"
60         expect \"$ROOT_PROMPT\"
61             send -- \"rm -f ~/hosts \r\"
62         expect \"$ROOT_PROMPT\"
63             send -- \"cat /etc/hosts \r\"
64         expect \"$ROOT_PROMPT\"
65     "
66     echo ""
67 done

  1 #!/bin/bash
  2 
  3 # PurPose:    Config Hadoop env
  4 # Author:    13Bear
  5 # Update:    2014-04-25 17:18
  6 
  7 # Exit value:
  8 # 0    All succeed
  9 # 1    Some failed
 10 # 98    expect not install
 11 # 99    Usage format error
 12 
 13 # Import env variable
 14 source ./conf/env.config
 15 
 16 # Prepare works
 17 # 1. hadoop-env.xml update with variables in env.config
 18 sed -i "s#JAVA_HOME=.*#JAVA_HOME=$JAVA_HOME#" $CONF_LOC/hadoop-env.sh
 19 
 20 # 2. core-site.xml update with variables in env.config
 21 #sed -i "/<name>hadoop.tmp.dir<\/name>/,/<\/value>/c \\
 22 #        <name>hadoop.tmp.dir</name>\n 23 #        <value>$HADOOP_TMP_DIR</value>" $CONF_LOC/core-site.xml
 24 sed -i "/hadoop\.tmp\.dir/{n;s#<value>.*</value>#<value>$HADOOP_TMP_DIR</value>#}" $CONF_LOC/core-site.xml
 25 #sed -i "s#hdfs://.*:9000#hdfs://$MASTER:9000#" $CONF_LOC/core-site.xml
 26 sed -i "/fs\.default\.name/{n;s#<value>.*</value>#<value>hdfs://$MASTER:9000</value>#}" $CONF_LOC/core-site.xml
 27 
 28 # 3. hdfs-site.xml update with variables in env.config
 29 #sed -i "/<name>dfs.replication<\/name>/,/<\/value>/c \\
 30 #        <name>dfs.replication<\/name>\n 31 #        <value>$HADOOP_DFS_REPLICATION</value>" $CONF_LOC/hdfs-site.xml
 32 sed -i "/dfs\.replication/{n;s#<value>.*</value>#<value>$HADOOP_DFS_REPLICATION</value>#}" $CONF_LOC/hdfs-site.xml
 33 
 34 # 4. mapred-site.xml update with variables in env.config
 35 #sed -i "s#http://.*:9001#http://$MASTER:9001#" $CONF_LOC/mapred-site.xml
 36 sed -i "/mapred\.job\.tracker/{n;s#<value>.*</value>#<value>http://$MASTER:9001</value>#}" $CONF_LOC/mapred-site.xml
 37 
 38 # 5. hadoop.env update with variables in env.config
 39 sed -i "s#HADOOP_HOME=.*#HADOOP_HOME=$HADOOP_HOME#" $CONF_LOC/hadoop.env
 40 
 41 # 6. tar all conf files - will scp to all hosts for hadoop configuration
 42 tar -czf conf.tar.gz conf
 43 
 44 # Start
 45 echo ""
 46 echo "+++++++++++++++++++++++++++++++++++++++++++++++ Hadoop Configuration Process +++++++++++++++++++++++++++++++++++++++++++++++"
 47 echo ""
 48 
 49 # Check the expect tool‘s installation
 50 if [ $EXPECTCHK != 1 ]; then
 51     echo "###############################################"
 52     echo "Please install the \"expect\" package first on all nodes to allow the script to run"
 53     echo "yum -y install expect"
 54     echo "###############################################"
 55     exit 98
 56 fi
 57 
 58 # Configuration Hadoop env on all hosts
 59 for host in $HOSTS
 60 do
 61     echo "###############################################"
 62     echo "Configuring Hadoop env on $host"
 63     echo "###############################################"
 64     echo "==========> 1.Copying Hadoop package to $host"
 65     expect -c "
 66         set timeout $EXPECT_TIMEOUT_BIG
 67         spawn scp $TOOLS_LOC/$HADOOP_PKGNAME root@$host:$HADOOP_INSTLOC
 68         expect \"yes/no\" {
 69             send \"yes\r\"
 70             expect \"password:\"
 71             send -- \"$ROOT_USERPWD\r\"
 72         } \"password:\" {
 73             send -- \"$ROOT_USERPWD\r\"
 74         }
 75         expect \"$ROOT_PROMPT\"
 76     "
 77     echo "==========> 2.Extracting Hadoop file to $HADOOP_INSTLOC on $host"
 78     expect -c "
 79         set timeout $EXPECT_TIMEOUT_BIG
 80         spawn ssh root@$host
 81         expect \"password:\"
 82             send -- \"$ROOT_USERPWD\r\"
 83         expect \"$ROOT_PROMPT\"
 84             send -- \"rm -rf $HADOOP_HOME \r\"
 85         expect \"$ROOT_PROMPT\"
 86             send -- \"mkdir $HADOOP_INSTLOC 2>/dev/null \r\"
 87         expect \"$ROOT_PROMPT\"
 88             send -- \"tar -xzf $HADOOP_INSTLOC/$HADOOP_PKGNAME -C $HADOOP_INSTLOC \r\"
 89         expect \"$ROOT_PROMPT\"
 90             send -- \"chown -R $HADOOP_USERNAME:$HADOOP_USERNAME $HADOOP_HOME \r\"
 91         expect \"$ROOT_PROMPT\"
 92             send -- \"rm -f $HADOOP_INSTLOC/$HADOOP_PKGNAME \r\"
 93         expect \"$ROOT_PROMPT\"
 94     "
 95     echo ""
 96     echo "==========> 3.Configuring Hadoop on $host"
 97     echo "=====> 3.0 Copying Hadoop config files conf.tar.gz to $host"
 98     expect -c "
 99         set timeout $EXPECT_TIMEOUT_BIG
100         spawn scp conf.tar.gz root@$host:~/
101         expect \"password:\"
102         send -- \"$ROOT_USERPWD\r\"
103         expect \"$ROOT_PROMPT\"
104     "
105     echo "=====> 3.1 Configing Hadoop env to /etc/profile  on $host"
106     expect -c "
107         set timeout $EXPECT_TIMEOUT_BIG
108         spawn ssh root@$host
109         expect \"password:\"
110             send -- \"$ROOT_USERPWD\r\"
111         expect \"$ROOT_PROMPT\"
112             send -- \"tar -xzf conf.tar.gz \r\"
113         expect \"$ROOT_PROMPT\"
114             send -- \"sed -i ‘/hadoop/Id‘ /etc/profile \r\"
115         expect \"$ROOT_PROMPT\"
116             send -- \"cat ~/conf/hadoop.env >> /etc/profile \r\"
117         expect \"$ROOT_PROMPT\"
118             send -- \"source /etc/profile \r\"
119         expect \"$ROOT_PROMPT\"
120             send -- \"hadoop version \r\"
121         expect \"$ROOT_PROMPT\"
122     "    
123     echo ""
124     echo "=====> 3.2 Configing hadoop-env.sh file on $host"
125     expect -c "
126         set timeout $EXPECT_TIMEOUT_BIG
127         spawn ssh root@$host
128         expect \"password:\"
129             send -- \"$ROOT_USERPWD\r\"
130         expect \"$ROOT_PROMPT\"
131             send -- \"cat ~/conf/hadoop-env.sh >> $HADOOP_CONFLOC/hadoop-env.sh \r\"
132         expect \"$ROOT_PROMPT\"
133             send -- \"tail -2 $HADOOP_CONFLOC/hadoop-env.sh \r\"
134         expect \"$ROOT_PROMPT\"
135     "    
136     echo ""
137     echo "=====> 3.3 Configing core-site.xml file on $host"
138     expect -c "
139         set timeout $EXPECT_TIMEOUT_BIG
140         spawn ssh root@$host
141         expect \"password:\"
142             send -- \"$ROOT_USERPWD\r\"
143         expect \"$ROOT_PROMPT\"
144             send -- \"sed -i ‘/<\\\/configuration>/d‘ $HADOOP_CONFLOC/core-site.xml \r\"
145         expect \"$ROOT_PROMPT\"
146             send -- \"cat ~/conf/core-site.xml >> $HADOOP_CONFLOC/core-site.xml \r\"
147         expect \"$ROOT_PROMPT\"
148             send -- \"cat $HADOOP_CONFLOC/core-site.xml | grep -v ‘^\\\s*\$‘ \r\"
149         expect \"$ROOT_PROMPT\"
150     "
151     echo ""
152     echo "=====> 3.4 Configing hdfs-site.xml file on $host"
153     expect -c "
154         set timeout $EXPECT_TIMEOUT_BIG
155         spawn ssh root@$host
156         expect \"password:\"
157             send -- \"$ROOT_USERPWD\r\"
158         expect \"$ROOT_PROMPT\"
159             send -- \"sed -i ‘/<\\\/configuration>/d‘ $HADOOP_CONFLOC/hdfs-site.xml \r\"
160         expect \"$ROOT_PROMPT\"
161             send -- \"cat ~/conf/hdfs-site.xml >> $HADOOP_CONFLOC/hdfs-site.xml \r\"
162         expect \"$ROOT_PROMPT\"
163             send -- \"cat $HADOOP_CONFLOC/hdfs-site.xml | grep -v ‘^\\\s*\$‘ \r\"
164         expect \"$ROOT_PROMPT\"
165     "
166     echo ""
167     echo "=====> 3.5 Configing mapred-site.xml file on $host"
168     expect -c "
169         set timeout $EXPECT_TIMEOUT_BIG
170         spawn ssh root@$host
171         expect \"password:\"
172             send -- \"$ROOT_USERPWD\r\"
173         expect \"$ROOT_PROMPT\"
174             send -- \"sed -i ‘/<\\\/configuration>/d‘ $HADOOP_CONFLOC/mapred-site.xml \r\"
175         expect \"$ROOT_PROMPT\"
176             send -- \"cat ~/conf/mapred-site.xml >> $HADOOP_CONFLOC/mapred-site.xml \r\"
177         expect \"$ROOT_PROMPT\"
178             send -- \"cat $HADOOP_CONFLOC/mapred-site.xml | grep -v ‘^\\\s*\$‘ \r\"
179         expect \"$ROOT_PROMPT\"
180     "
181     echo ""
182     echo "=====> 3.6 Configing masters file on $host"
183     expect -c "
184         set timeout $EXPECT_TIMEOUT_BIG
185         spawn ssh root@$host
186         expect \"password:\"
187             send -- \"$ROOT_USERPWD\r\"
188         expect \"$ROOT_PROMPT\"
189             send -- \"cat ~/conf/masters > $HADOOP_CONFLOC/masters \r\"
190         expect \"$ROOT_PROMPT\"
191             send -- \"cat $HADOOP_CONFLOC/masters \r\"
192         expect \"$ROOT_PROMPT\"
193     "
194     echo ""
195     echo "=====> 3.7 Configing slaves file on $host"
196     expect -c "
197         set timeout $EXPECT_TIMEOUT_BIG
198         spawn ssh root@$host
199         expect \"password:\"
200             send -- \"$ROOT_USERPWD\r\"
201         expect \"$ROOT_PROMPT\"
202             send -- \"cat ~/conf/slaves > $HADOOP_CONFLOC/slaves \r\"
203         expect \"$ROOT_PROMPT\"
204             send -- \"cat $HADOOP_CONFLOC/slaves \r\"
205         expect \"$ROOT_PROMPT\"
206             send -- \"rm -rf conf; rm -f conf.tar.gz \r\"
207         expect \"$ROOT_PROMPT\"
208     "
209     echo ""
210 done
211 
212 # Remove conf tar file in local
213 rm -f conf.tar.gz

 1 #!/bin/bash
 2 
 3 # Purpose:    Uninstall Hadoop
 4 # Author:       13Bear
 5 # Update:       2014-04-25 17:18
 6 
 7 # Exit value:
 8 # 0     Uninstall succeed
 9 # 1     Some hosts can‘t be reached
10 # 2    Hadoop is running
11 
12 # Import env variable
13 source ./conf/env.config
14 
15 # Make sure all the hosts can be reach
16 ./pingAll.sh
17 if [ $? -eq 1 ]; then
18     echo "###############################################"
19     echo -e "\033[31mSome nodes can‘t be reached! Please check them!\033[0m"
20     echo "###############################################"
21     exit 1
22 fi
23 
24 # Check the hadoop running status
25 ./hadoopStatusCheck.sh
26 if [ $? -eq 1 ]; then
27     exit 2
28 fi
29 
30 # Start uninstall
31 echo ""
32 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
33 echo "++++++++++++++++++++++++++++++++++++++++++++++ Start Uninstall +++++++++++++++++++++++++++++++++++++++++++++++"
34 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
35 echo ""
36 ./unhadoopAll.sh
37 ./unhostsAll.sh
38 ./unjavaAll.sh
39 ./userdelAll.sh
40 echo ""
41 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
42 echo "++++++++++++++++++++++++++++++++++++++++++++++++ End Uninstall +++++++++++++++++++++++++++++++++++++++++++++++"
43 echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
44 echo ""
45 exit 0

 1 #!/bin/bash
 2 
 3 # PurPose:    Config Hadoop env
 4 # Author:    13Bear
 5 # Update:    2014-04-25 17:18
 6 
 7 # Exit value:
 8 # 0    All succeed
 9 # 1    Some failed
10 # 98    expect not install
11 # 99    Usage format error
12 
13 # Import env variable
14 source ./conf/env.config
15 
16 # Start
17 echo ""
18 echo "+++++++++++++++++++++++++++++++++++++++++++++++ Hadoop Status Check +++++++++++++++++++++++++++++++++++++++++++++++"
19 echo ""
20 
21 # Check the expect tool‘s installation
22 if [ $EXPECTCHK != 1 ]; then
23     echo "###############################################"
24     echo "Please install the \"expect\" package first on all nodes to allow the script to run"
25     echo "yum -y install expect"
26     echo "###############################################"
27     exit 98
28 fi
29 
30 # Checking Hadoop Status on Master
31 hadoop_status=0
32 echo "###############################################"
33 echo "Hadoop Status Check on Master -> $MASTER"
34 echo "###############################################"
35 expect -c "
36     set timeout $EXPECT_TIMEOUT_BIG
37     spawn ssh root@$MASTER
38     expect \"yes/no\" {
39         send \"yes\r\"
40         expect \"password:\"
41         send -- \"$ROOT_USERPWD\r\"
42     } \"password:\" {
43         send -- \"$ROOT_USERPWD\r\"
44     }
45     expect \"$ROOT_PROMPT\"
46         send -- \"jps\r\"
47     expect \"NameNode\" {
48         expect \"$ROOT_PROMPT\"
49         exit 1
50     } \"$ROOT_PROMPT\" {
51         exit 0
52     }
53 "
54 hadoop_status=$?
55 if [ $hadoop_status -eq 1 ]; then
56     echo ""
57     echo "###############################################"
58     echo -e "\033[31mHadoop is Running!\033[0m"
59     echo "###############################################"
60     exit 1
61 elif [ $hadoop_status -eq 0 ]; then
62     echo ""
63     echo "###############################################"
64     echo -e "\033[32mHadoop is Down!\033[0m"
65     echo "###############################################"
66     exit 0
67 fi

 1 #!/bin/bash
 2 
 3 # PurPose:    Uninstall Hadoop env
 4 # Author:    13Bear
 5 # Update:    2014-04-25 17:18
 6 
 7 # Exit value:
 8 # 0    All succeed
 9 # 1    Some failed
10 # 98    expect not install
11 # 99    Usage format error
12 
13 # Import env variable
14 source ./conf/env.config
15 
16 # Start
17 echo ""
18 echo "+++++++++++++++++++++++++++++++++++++++++++++++ Uninstall Hadoop Process +++++++++++++++++++++++++++++++++++++++++++++++"
19 echo ""
20 
21 # Check the expect tool‘s installation
22 if [ $EXPECTCHK != 1 ]; then
23     echo "###############################################"
24     echo "Please install the \"expect\" package first on all nodes to allow the script to run"
25     echo "yum -y install expect"
26     echo "###############################################"
27     exit 98
28 fi
29 
30 # Uninstall Hadoop on all hosts
31 for host in $HOSTS
32 do
33     echo "###############################################"
34     echo "Uninstalling Hadoop env on $host"
35     echo "###############################################"
36     expect -c "
37         set timeout $EXPECT_TIMEOUT_BIG
38         spawn ssh root@$host
39         expect \"yes/no\" {
40             send \"yes\r\"
41             expect \"password:\"
42             send -- \"$ROOT_USERPWD\r\"
43         } \"password:\" {
44             send -- \"$ROOT_USERPWD\r\"
45         }
46         expect \"$ROOT_PROMPT\"
47             send -- \"rm -rf $HADOOP_HOME \r\"
48         expect \"$ROOT_PROMPT\"
49             send -- \"sed -i ‘/hadoop/Id‘ /etc/profile \r\"
50         expect \"$ROOT_PROMPT\"
51             send -- \"rm -rf /tmp/hadoop-*;rm -rf /tmp/hsperfdata_*;rm -rf /tmp/Jetty_0_0_0_0_* \r\"
52         expect \"$ROOT_PROMPT\"
53     "
54     echo ""
55 done

 1 #!/bin/bash
 2 
 3 # PurPose:    Remove hadoop hosts in /etc/hosts
 4 # Author:    13Bear
 5 # Update:    2014-04-25 17:18
 6 
 7 # Exit value:
 8 # 0    All succeed
 9 # 1    Some failed
10 # 98    expect not install
11 # 99    Usage format error
12 
13 # Import env variable
14 source ./conf/env.config
15 
16 # Start
17 echo ""
18 echo "+++++++++++++++++++++++++++++++++++++++++++++++ Remove hadoop hosts in /etc/hosts Process +++++++++++++++++++++++++++++++++++++++++++++++"
19 echo ""
20 
21 # Check the expect tool‘s installation
22 if [ $EXPECTCHK != 1 ]; then
23     echo "###############################################"
24     echo "Please install the \"expect\" package first on all nodes to allow the script to run"
25     echo "yum -y install expect"
26     echo "###############################################"
27     exit 98
28 fi
29 
30 # Remove hadoop hosts in /etc/hosts for all hosts
31 for host in $HOSTS
32 do
33     echo "###############################################"
34     echo "Removing hadoop hosts in /etc/hosts on $host"
35     echo "###############################################"
36     # sed 命令查找删除的关键字 hadoop 一定要在 hosts 文件中包含，不然 sed 删除失败
37     expect -c "
38         set timeout $EXPECT_TIMEOUT_BIG
39         spawn ssh root@$host
40         expect \"yes/no\" {
41             send \"yes\r\"
42             expect \"password:\"
43             send -- \"$ROOT_USERPWD\r\"
44         } \"password:\" {
45             send -- \"$ROOT_USERPWD\r\"
46         }
47         expect \"$ROOT_PROMPT\"
48             send -- \"sed -i ‘/hadoop/Id‘ /etc/hosts \r\"
49         expect \"$ROOT_PROMPT\"
50     "
51     echo ""
52 done

 1 #!/bin/bash
 2 
 3 # PurPose:    Uninstall Java env
 4 # Author:    13Bear
 5 # Update:    2014-04-25 17:18
 6 
 7 # Exit value:
 8 # 0    All succeed
 9 # 1    Some failed
10 # 98    expect not install
11 # 99    Usage format error
12 
13 # Import env variable
14 source ./conf/env.config
15 
16 # Start
17 echo ""
18 echo "+++++++++++++++++++++++++++++++++++++++++++++++ Uninstall Java Process +++++++++++++++++++++++++++++++++++++++++++++++"
19 echo ""
20 
21 # Check the expect tool‘s installation
22 if [ $EXPECTCHK != 1 ]; then
23     echo "###############################################"
24     echo "Please install the \"expect\" package first on all nodes to allow the script to run"
25     echo "yum -y install expect"
26     echo "###############################################"
27     exit 98
28 fi
29 
30 # Uninstall Java for all hosts
31 for host in $HOSTS
32 do
33     echo "###############################################"
34     echo "Uninstalling Java env on $host"
35     echo "###############################################"
36     expect -c "
37         set timeout $EXPECT_TIMEOUT_BIG
38         spawn ssh root@$host
39         expect \"yes/no\" {
40             send \"yes\r\"
41             expect \"password:\"
42             send -- \"$ROOT_USERPWD\r\"
43         } \"password:\" {
44             send -- \"$ROOT_USERPWD\r\"
45         }
46         expect \"$ROOT_PROMPT\"
47             send -- \"rm -rf $JAVA_HOME \r\"
48         expect \"$ROOT_PROMPT\"
49             send -- \"sed -i ‘/java/Id‘ /etc/profile \r\"
50         expect \"$ROOT_PROMPT\"
51     "
52     echo ""
53 done

 1 #!/bin/bash
 2 
 3 # Purpose:    Config the hadoop env‘s ssh no password login
 4 # Author:       13Bear
 5 # Update:       2014-04-25 17:18
 6 
 7 # Exit value:
 8 # 0     All succeed
 9 # 1     Some failed
10 # 98    expect not install
11 # 99    Usage format error
12 
13 # Import env variable
14 source ./conf/env.config
15 
16 # Start
17 echo ""
18 echo "+++++++++++++++++++++++++++++++++++++++++++++++ Userdel Process +++++++++++++++++++++++++++++++++++++++++++++++" 
19 echo ""
20 
21 # Check the expect tool‘s installation
22 if [ $EXPECTCHK != 1 ]; then
23         echo "###############################################"
24         echo "Please install the \"expect\" package first on all nodes to allow the script to run"
25         echo "yum -y install expect"
26         echo "###############################################"
27         exit 98
28 fi
29 
30 # Delete hadoop user for every host
31 for host in $HOSTS
32 do
33     echo "###############################################"
34     echo "Deleting hadoop user \"$HADOOP_USERNAME\" for host $host"
35     echo "###############################################"
36     expect -c "
37         set timeout $EXPECT_TIMEOUT
38         spawn ssh root@$host
39         expect \"yes/no\" {
40             send \"yes\r\"
41             expect \"password:\"
42             send -- \"$ROOT_USERPWD\r\"
43         } \"password:\" {
44             send -- \"$ROOT_USERPWD\r\"
45         }
46         expect \"$ROOT_PROMPT\"
47             send -- \"userdel -r $HADOOP_USERNAME\r\"
48         expect \"$ROOT_PROMPT\"
49     "
50     echo ""
51 done