Spark学习之路 （五）Spark伪分布式安装

[root@hadoop1 soft]# tar -zxvf jdk-8u73-linux-x64.tar.gz -C /usr/local/

[root@hadoop1 soft]# vi /etc/profile

#JAVA
export JAVA_HOME=/usr/local/jdk1.8.0_73
export CLASSPATH=$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib 
export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH:$HOME/bin 

[root@hadoop1 soft]# java -version

[hadoop@hadoop1 ~]$ ssh-keygen -t rsa

[hadoop@hadoop1 ~]$ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

[hadoop@hadoop1 ~]$ chmod 600 ~/.ssh/authorized_keys 

[root@hadoop1 ~]$ vi /etc/hosts

[hadoop@hadoop1 ~]$ ssh hadoop1

[hadoop@hadoop1 ~]$ tar -zxvf hadoop-2.7.5-centos-6.7.tar.gz -C apps/

[hadoop@hadoop1 ~]$ cd apps/
[hadoop@hadoop1 apps]$ ll
总用量 4
drwxr-xr-x. 9 hadoop hadoop 4096 12月 24 13:43 hadoop-2.7.5
[hadoop@hadoop1 apps]$ ln -s hadoop-2.7.5/ hadoop

[hadoop@hadoop1 hadoop]$ vi hadoop-env.sh 

export JAVA_HOME=/usr/local/jdk1.8.0_73 

[hadoop@hadoop1 hadoop]$ vi core-site.xml 

<configuration>
        <property>
                <name>fs.defaultFS</name>
                <value>hdfs://hadoop1:9000</value>
        </property>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>/home/hadoop/data/hadoopdata</value>
        </property>
</configuration>

[hadoop@hadoop1 hadoop]$ vi hdfs-site.xml 

        <property>
                <name>dfs.namenode.name.dir</name>
                <value>/home/hadoop/data/hadoopdata/name</value>
                <description>为了保证元数据的安全一般配置多个不同目录</description>
        </property>

        <property>
                <name>dfs.datanode.data.dir</name>
                <value>/home/hadoop/data/hadoopdata/data</value>
                <description>datanode 的数据存储目录</description>
        </property>

        <property>
                <name>dfs.replication</name>
                <value>2</value>
                <description>HDFS 的数据块的副本存储个数, 默认是3</description>
        </property>    

[hadoop@hadoop1 hadoop]$ cp mapred-site.xml.template mapred-site.xml
[hadoop@hadoop1 hadoop]$ vi mapred-site.xml

<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
</configuration>

[hadoop@hadoop1 hadoop]$ vi yarn-site.xml 

 　　　　<property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
                <description>YARN 集群为 MapReduce 程序提供的 shuffle 服务</description>
        </property>

[hadoop@hadoop1 ~]$ vi .bashrc

#HADOOP_HOME export HADOOP_HOME=/home/hadoop/apps/hadoop-2.7.5
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:

[hadoop@hadoop1 bin]$ source ~/.bashrc 

[hadoop@hadoop1 ~]$ hadoop version

[hadoop@hadoop1 ~]$ mkdir -p /home/hadoop/data/hadoopdata/name
[hadoop@hadoop1 ~]$ mkdir -p /home/hadoop/data/hadoopdata/data

[hadoop@hadoop1 ~]$ hadoop namenode -format

[hadoop@hadoop1 ~]$ start-dfs.sh [hadoop@hadoop1 ~]$ start-yarn.sh

[root@hadoop1 hadoop]# tar -zxvf scala-2.11.8.tgz -C /usr/local/

[root@hadoop1 hadoop]# vi /etc/profile

#Scala
export SCALA_HOME=/usr/local/scala-2.11.8
export PATH=$SCALA_HOME/bin:$PATH

[root@hadoop1 scala-2.11.8]# source /etc/profile

[root@hadoop1 ~]# scala -version

[hadoop@hadoop1 ~]$ tar -zxvf spark-2.3.0-bin-hadoop2.7.tgz -C apps/

[hadoop@hadoop1 ~]$ cd apps/
[hadoop@hadoop1 apps]$ ls
hadoop  hadoop-2.7.5  spark-2.3.0-bin-hadoop2.7
[hadoop@hadoop1 apps]$ ln -s spark-2.3.0-bin-hadoop2.7/ spark

[hadoop@hadoop1 apps]$ cd spark/conf/

[hadoop@hadoop1 conf]$ cp spark-env.sh.template spark-env.sh
[hadoop@hadoop1 conf]$ vi spark-env.sh

export JAVA_HOME=/usr/local/jdk1.8.0_73
export SCALA_HOME=/usr/share/scala-2.11.8
export HADOOP_HOME=/home/hadoop/apps/hadoop-2.7.5
export HADOOP_CONF_DIR=/home/hadoop/apps/hadoop-2.7.5/etc/hadoop
export SPARK_MASTER_IP=hadoop1
export SPARK_MASTER_PORT=7077

[hadoop@hadoop1 conf]$ vi ~/.bashrc 

#SPARK_HOME
export SPARK_HOME=/home/hadoop/apps/spark
export PATH=$PATH:$SPARK_HOME/bin

[hadoop@hadoop1 conf]$ source ~/.bashrc

[hadoop@hadoop1 ~]$  ~/apps/spark/sbin/start-all.sh