hadoop 集群搭建

1、解压jdk、hadoop

cd /opt/software/
tar -xvf hadoop-3.1.3.tar.gz
tar -xvf jdk-8u212-linux-x64.tar.gz

mv hadoop-3.1.3 /opt/module/
mv jdk1.8.0_212 /opt/module/

2、profile配置

vim /etc/profile

# JAVA
export JAVA_HOME=/opt/module/jdk1.8.0_212
export PATH=$PATH:$JAVA_HOME/bin

# HADOOP
export HADOOP_HOME=/opt/module/hadoop-3.1.3
export PATH=$PATH:$HADOOP_HOME/bin

# HDFS YARN USER 
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

scp /etc/profile slave1:/etc/profile
scp /etc/profile slave2:/etc/profile

source /etc/profile

测试是否生效

java -version
hadoop version

3、core 配置

vim $HADOOP_HOME/etc/hadoop/core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <!-- 指定 NameNode 的地址 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://master:8020</value>
    </property>
    <!-- 指定 hadoop 数据的存储目录 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/module/hadoop-3.1.3/data</value>
    </property>
    <!-- 配置 HDFS 网页登录使用的静态用户为 root -->
    <property>
        <name>hadoop.http.staticuser.user</name>
        <value>root</value>
    </property>
</configuration>

4、hdfs 配置

vim $HADOOP_HOME/etc/hadoop/hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <!-- nn web 端访问地址-->
    <property>
        <name>dfs.namenode.http-address</name>
        <value>master:9870</value>
    </property>
    <!-- 2nn web 端访问地址-->
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>slave1:9868</value>
    </property>
</configuration>

5、yarn 配置

vim $HADOOP_HOME/etc/hadoop/yarn-site.xml

<configuration>
    <!-- 指定 MR 走 shuffle -->
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <!-- 指定 ResourceManager 的地址-->
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>master</value>
    </property>
    <!-- 环境变量的继承 -->
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CO
            NF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAP
            RED_HOME</value>
    </property>
    <!-- 开启日志聚集功能 -->
    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>
            <!-- 设置日志聚集服务器地址 -->
    <property>
    <name>yarn.log.server.url</name>
    <value>http://master:19888/jobhistory/logs</value>
    </property>
            <!-- 设置日志保留时间为 7 天 -->
    <property>
    <name>yarn.log-aggregation.retain-seconds</name>
    <value>604800</value>
    </property>
</configuration>

6、mapred 配置

vim $HADOOP_HOME/etc/hadoop/mapred-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <!-- 指定 MapReduce 程序运行在 Yarn 上 -->
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <!-- 历史服务器端地址 -->
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>master:10020</value>
    </property>
    <!-- 历史服务器 web 端地址 -->
    <property>
    <name>mapreduce.jobhistory.webapp.address</name>
    <value>master:19888</value>
    </property>
</configuration>

7、hadoop env

vim $HADOOP_HOME/etc/hadoop/hadoop-env.sh
# 增加 JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_212

8、workers

vim $HADOOP_HOME/etc/hadoop/workers
master
slave1
slave2

9、分发配置

scp -r $HADOOP_HOME/etc/hadoop/ slave1:$HADOOP_HOME/etc/
scp -r $HADOOP_HOME/etc/hadoop/ slave2:$HADOOP_HOME/etc/

10、初始化hdfs

hdfs namenode -format

11、启动集群

HDFS

启动
$HADOOP_HOME/sbin/start-dfs.sh
关闭
$HADOOP_HOME/sbin/stop-dfs.sh
访问地址
http://192.168.25.100:9870/

YARN

启动
$HADOOP_HOME/sbin/start-yarn.sh
关闭
$HADOOP_HOME/sbin/stop-yarn.sh
访问地址
http://192.168.25.101:8088/

历史服务器

启动
mapred --daemon start historyserver
关闭
mapred --daemon stop historyserver
访问地址
http://192.168.25.100:19888/

高可用（HA）的配置文件（可选）

参考 https://blog.csdn.net/qq_39604679/article/details/125035946

export HDFS_JOURNALNODE_USER=root
export JAVA_HOME=/opt/module/jdk
export HDFS_ZKFC_USER=root

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
<!-- 把多个NameNode的地址组装成一个集群，集群名字随便取 -->
<property>
        <name>fs.defaultFS</name>
    <value>hdfs://mycluster</value>
</property>
<!-- 指定hadoop运行时产生文件的存储目录 -->
<property>
        <name>hadoop.tmp.dir</name>
    <value>/opt/module/hadoop/tmp</value>
</property>
<!-- 指定要连接的zkServer地址 -->
<property>
        <name>ha.zookeeper.quorum</name>
    <value>master:2181,slave1:2181,slave2:2181</value>
</property>
</configuration>  

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
<!-- namenode存储目录 -->
<property>
	<name>dfs.namenode.name.dir</name>
    <value>/opt/module/hadoop/name</value>
</property>
<!-- datanode数据存储目录 -->
<property>
	<name>dfs.datanode.data.dir</name>
    <value>/opt/module/hadoop/data</value>
</property>
<!-- journalnode数据存储目录 -->
<property>
	<name>dfs.journalnode.edits.dir</name>
    <value>/opt/module/hadoop/jn</value>
</property>
<!-- 完全分布式集群名称 -->
<property>
	<name>dfs.nameservices</name>
    <value>mycluster</value>
</property>
<!-- 集群中的namenode节点都有哪些 -->
<property>
	<name>dfs.ha.namenodes.mycluster</name>
    <value>nn1,nn2,nn3</value>
</property>
<!-- namenode的RPC通信地址 -->
<property>
	<name>dfs.namenode.rpc-address.mycluster.nn1</name>
    <value>master:9000</value>
</property>
<property>
	<name>dfs.namenode.rpc-address.mycluster.nn2</name>
    <value>slave1:9000</value>
</property>
<property>
	<name>dfs.namenode.rpc-address.mycluster.nn3</name>
    <value>slave2:9000</value>
</property>
<!-- namenode的http通信地址 -->
<property>
	<name>dfs.namenode.http-address.mycluster.nn1</name>
    <value>master:9870</value>
</property>
<property>
	<name>dfs.namenode.http-address.mycluster.nn2</name>
    <value>slave1:9870</value>
</property>
<property>
	<name>dfs.namenode.http-address.mycluster.nn3</name>
    <value>slave2:9870</value>
</property>
<!-- 指定namenode元数据在journalnode上的存放位置，journalnode用于同步主备namenode之间的edits文件 -->
<property>
	<name>dfs.namenode.shared.edits.dir</name>
    <value>qjournal://master:8485;slave1:8485;slave2:8485/mycluster</value>
</property>
<!-- 访问代理类：client用于确定哪个NameNode为Active -->
<property>
	<name>dfs.client.failover.proxy.provider.mycluster</name>
    <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<!-- 配置隔离机制，即同一时刻只能有一台服务器对外响应 -->
<property>  
 <name>dfs.ha.fencing.methods</name>  
     <value>
        sshfence
        shell(/bin/true)
    </value>
</property>
<!-- 使用隔离机制时需要ssh秘钥登陆 -->
<property>
	<name>dfs.ha.fencing.ssh.private-key-files</name>
    <value>/root/.ssh/id_rsa</value>
</property>
<!-- 启用namenode故障自动转移 -->
<property>
	<name>dfs.ha.automatic-failover.enabled</name>
    <value>true</value>
</property>
</configuration>


<?xml version="1.0"?>

<configuration>

<!-- nodemanager获取数据的方式 -->
<property>
	<name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
</property>
<!-- 启用resourcemanager ha -->
<property>
	<name>yarn.resourcemanager.ha.enabled</name>
    <value>true</value>
</property>
<!-- 声明resourcemanager的地址 -->
<property>
	<name>yarn.resourcemanager.cluster-id</name>
    <value>cluster-yarn1</value>
</property>
<!-- 指定resourcemanager的逻辑列表 -->
<property>
	<name>yarn.resourcemanager.ha.rm-ids</name>
    <value>rm1,rm2,rm3</value>
</property>

<!-- rm1的配置=-========= -->
<!-- 指定rm1的主机名 -->
<property>
	<name>yarn.resourcemanager.hostname.rm1</name>
    <value>master</value>
</property>
<!-- 指定rm1的web端地址 -->
<property>
	<name>yarn.resourcemanager.webapp.address.rm1</name>
    <value>master:8088</value>
</property>

<!-- rm2的配置=-========= -->
<!-- 指定rm2的主机名 -->
<property>
	<name>yarn.resourcemanager.hostname.rm2</name>
    <value>slave1</value>
</property>
<!-- 指定rm2的web端地址 -->
<property>
	<name>yarn.resourcemanager.webapp.address.rm2</name>
    <value>slave1:8088</value>
</property>

<!-- rm3的配置=-========= -->
<!-- 指定rm3的主机名 -->
<property>
	<name>yarn.resourcemanager.hostname.rm3</name>
    <value>slave2</value>
</property>
<!-- 指定rm3的web端地址 -->
<property>
	<name>yarn.resourcemanager.webapp.address.rm3</name>
    <value>slave2:8088</value>
</property>

<!-- 指定zookeeper集群地址 -->
<property>
	<name>hadoop.zk.address</name>
    <value>master:2181,slave1:2181,worker2:2181</value>
</property>

<!-- 启用自动恢复 -->
<property>
	<name>yarn.resourcemanager.recovery.enabled</name>
    <value>true</value>
</property>

<!-- 指定resourcemanager的状态信息存储在zookeeper集群 -->
<property>
	<name>yarn.resourcemanager.store.class</name>
    <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>

<!-- 忽略虚拟内存检查 -->
<property>
    <name>yarn.nodemanager.vmem-check-enabled</name>
    <value>false</value>
</property>
</configuration>


dataLogDir=/opt/module/zookeeper/logs
dataDir=/opt/module/zookeeper/data
# the port at which the clients will connect
clientPort=2181

server.1=master:2888:3888
server.2=slave1:2888:3888
server.3=slave2:2888:3888

评论