技术

Hadoop 集群部署

1、解压jdk、hadoop

cd /opt/software/
mkdir /opt/module/
tar -xvf hadoop-3.1.3.tar.gz
tar -xvf jdk-8u212-linux-x64.tar.gz

mv hadoop-3.1.3 /opt/module/
mv jdk1.8.0_212 /opt/module/

2、profile配置

vim /etc/profile

# JAVA 
export JAVA_HOME=/opt/module/jdk1.8.0_212
export PATH=$PATH:$JAVA_HOME/bin

# HADOOP
export HADOOP_HOME=/opt/module/hadoop-3.1.3
export PATH=$PATH:$HADOOP_HOME/bin

# HDFS YARN USER 
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

scp /etc/profile slave1:/etc/
scp /etc/profile slave2:/etc/

source /etc/profile

测试是否生效

java -version
hadoop version

3、core 配置

vim $HADOOP_HOME/etc/hadoop/core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->

<configuration>
<!--NameNode-->
<property>
<name>fs.defaultFS</name>
<value>hdfs://master:8020</value>
</property>

<!--数据存储位置-->
<property>
<name>hadoop.tmp.dir</name>
<value>/opt/module/hadoop-3.1.3/data</value>
</property>

<!--配置用户-->
<property>
<name>hadoop.http.staticuser.user</name>
<value>root</value>
</property>
</configuration>

4、hdfs 配置

vim $HADOOP_HOME/etc/hadoop/hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->

<configuration>

<!-- NameNode对外暴露的Web端访问地址 -->
<property>
  <name>dfs.namenode.http-address</name>
  <value>master:9870</value>
</property>

<!-- 2NN 对外暴露的Web端访问地址（2NN安装在hadoop104上） -->
<property>
  <name>dfs.namenode.secondary.http-address</name>
  <value>slave2:9868</value>
</property>

</configuration>

5、yarn 配置

vim $HADOOP_HOME/etc/hadoop/yarn-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <!-- 指定MR走shuffle -->
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>

    <!-- 指定ResourceManager的地址-->
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>slave1</value>
    </property>

	<property>
	  <name>yarn.resourcemanager.webapp.address</name>
	  <value>0.0.0.0:8088</value>
	</property>
    <!-- 环境变量的继承 -->
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
 <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
    </property>

    <!-- 开启日志聚集功能 -->
    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>
    <!-- 设置日志聚集服务器地址 -->
    <property>  
        <name>yarn.log.server.url</name>  
        <value>http://master:19888/jobhistory/logs</value>
    </property>
    <!-- 设置日志保留时间为7天 -->
    <property>
        <name>yarn.log-aggregation.retain-seconds</name>
        <value>604800</value>
    </property>
</configuration>

6、mapred 配置

vim $HADOOP_HOME/etc/hadoop/mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->

<configuration>

<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>

<property>
    <name>yarn.app.mapreduce.am.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/module/hadoop-3.1.3</value>
</property>
<property>
    <name>mapreduce.map.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/module/hadoop-3.1.3</value>
</property>
<property>
    <name>mapreduce.reduce.env</name>
    <value>HADOOP_MAPRED_HOME=/opt/module/hadoop-3.1.3</value>
</property>

</configuration>

7、修改 hadoop env

vim $HADOOP_HOME/etc/hadoop/hadoop-env.sh
# 增加 JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_212

8、分发配置

vim $HADOOP_HOME/etc/hadoop/workers
master
slave1
slave2

scp -r $HADOOP_HOME/etc/hadoop/ slave1:$HADOOP_HOME/etc/
scp -r $HADOOP_HOME/etc/hadoop/ slave2:$HADOOP_HOME/etc/

9、初始化hdfs

mkdir $HADOOP_HOME/data
hdfs namenode -format

10、启动集群

$HADOOP_HOME/sbin/start-all.sh
# 结束集群 $HADOOP_HOME/sbin/stop-all.sh

# ResourceManager 单独启动 在 slave1 中运行
$HADOOP_HOME/sbin/start-yarn.sh

11、查看进程

jps
Comment