Debian下Hadoop集群安装

news/2025/3/19 7:19:39/

Debian下Hadoop集群安装

依赖安装

jdk 8

sudo apt-get update && sudo apt-get install -y wget apt-transport-https
wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | sudo tee /etc/apt/keyrings/adoptium.asc
echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://mirrors.tuna.tsinghua.edu.cn/Adoptium/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | sudo tee /etc/apt/sources.list.d/adoptium.list
sudo apt-get update
sudo apt-get install -y temurin-8-jdk

hadoop

mkdir -p /root/packages
wget -P /root/packages https://dlcdn.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6-aarch64.tar.gz
tar -zxvf /root/packages/hadoop-3.3.6-aarch64.tar.gz -C /usr/local

配置环境变量

export JAVA_HOME=/usr/lib/jvm/temurin-8-jdk-amd64
export HADOOP_HOME="/usr/local/hadoop-3.3.6"
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

环境配置

设置hosts

127.0.0.1       localhost
192.168.50.201  node1.node1.com node1
192.168.50.202  node2.node2.com node2
192.168.50.203  node3.node3.com node3# The following lines are desirable for IPv6 capable hosts
::1     localhost ip6-localhost ip6-loopback
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters

生成SSH rsa

ssh-keygen -t rsa -C "node1@example.com"

允许root登录

vim /etc/ssh/sshd_config

修改PermitRootLogin yes

复制ssh到其他主机

ssh-copy-id node1
ssh-copy-id node2
ssh-copy-id node3

至少需要完成从node1 -> node1,node2,node3的免密登录

Hadoop配置

编辑Hadoop配置文件

hadoop-env.sh

vim $HADOOP_HOME/etc/hadoop/hadoop-env.sh

在文件末尾加上以下内容

# 配置JAVA_HOME
export JAVA_HOME=$JAVA_HOME# 设置用户以执行对应角色shell命令
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

core-site.xml

vim $HADOOP_HOME/etc/hadoop/core-site.xml

在configuration标签中添加以下内容

<!-- 默认文件系统的名称。通过URI中schema区分不同文件系统 -->
<!-- file://本地文件系统 hdfs://hadoop分布式文件系统 -->
<!-- gfs://google文件系统 -->
<!-- hdfs文件系统访问地址：http://node1:8020 -->
<property><name>fs.defaultFS</name><value>hdfs://node1:8020</value>
</property><!-- 设置Hadoop本地保存数据路径 -->
<property><name>hadoop.tmp.dir</name><value>/export/data/hadoop</value>
</property><!-- 设置HDFS web UI用户身份 -->
<property><name>hadoop.http.staticuser.user</name><value>root</value>
</property>

mapred-site.xml

vim $HADOOP_HOME/etc/hadoop/mapred-site.xml

在configuration标签中添加以下内容

<!-- 设置MR程序默认运行模式： yarn集群模式 local本地模式 -->
<property><name>mapreduce.framework.name</name><value>yarn</value>
</property><!-- MR程序历史服务地址 -->
<property><name>mapreduce.jobhistory.address</name><value>node1:10020</value>
</property><!-- MR程序历史服务器web端地址 -->
<property><name>mapreduce.jobhistory.webapp.address</name><value>node1:19888</value>
</property><property><name>yarn.app.mapreduce.am.env</name><value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
</property><property><name>mapreduce.map.env</name><value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
</property><property><name>mapreduce.reduce.env</name><value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
</property>

yarn-site.xml

vim $HADOOP_HOME/etc/hadoop/yarn-site.xml

在configuration标签中添加以下内容

<!-- 设置YARN集群主角色运行机器位置 -->
<property><name>yarn.resourcemanager.hostname</name><value>node1</value>
</property><!-- ModeManager上运行的附属服务，需配置成mapreduce_shuffle才可运行程序。 -->
<property><name>yarn.nodemanager.aux-services</name><value>mapreduce_shuffle</value>
</property><!-- 每个容器请求的最小内存资源（MB为单位） -->
<property><name>yarn.scheduler.minimum-allocation-mb</name><value>256</value>
</property><!-- 每个容器请求的最大内存资源（MB为单位） -->
<property><name>yarn.scheduler.maximum-allocation-mb</name><value>512</value>
</property>

workers

vim $HADOOP_HOME/etc/hadoop/workers

在workers文件中添加主机名称或IP

node1
node2
node3

Hadoop启动

NameNode format(格式化操作)

首次启动HDFS时，必须对其进行格式化操作

format本质上是初始化工作，进行HDFS清理和准备工作

hdfs namenode -format

当格式化日志中出现以下内容说明格式化成功

STARTUP_MSG:   build = https://github.com/apache/hadoop.git -r 1be78238728da9266a4f88195058f08fd012bf9c; compiled by 'ubuntu' on 2023-06-18T23:15Z
STARTUP_MSG:   java = 1.8.0_382
************************************************************/
2023-09-12 21:18:41,575 INFO namenode.NameNode: registered UNIX signal handlers for [TERM, HUP, INT]
2023-09-12 21:18:41,758 INFO namenode.NameNode: createNameNode [-format]
2023-09-12 21:18:42,039 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2023-09-12 21:18:43,011 INFO namenode.NameNode: Formatting using clusterid: CID-fcf657e4-d0df-4a9a-8f7d-a2a8f0a910df
2023-09-12 21:18:43,082 INFO namenode.FSEditLog: Edit logging is async:true
2023-09-12 21:18:43,167 INFO namenode.FSNamesystem: KeyProvider: null
2023-09-12 21:18:43,173 INFO namenode.FSNamesystem: fsLock is fair: true
2023-09-12 21:18:43,179 INFO namenode.FSNamesystem: Detailed lock hold time metrics enabled: false
2023-09-12 21:18:43,229 INFO namenode.FSNamesystem: fsOwner                = root (auth:SIMPLE)
2023-09-12 21:18:43,233 INFO namenode.FSNamesystem: supergroup             = supergroup
2023-09-12 21:18:43,237 INFO namenode.FSNamesystem: isPermissionEnabled    = true
2023-09-12 21:18:43,238 INFO namenode.FSNamesystem: isStoragePolicyEnabled = true
2023-09-12 21:18:43,240 INFO namenode.FSNamesystem: HA Enabled: false
2023-09-12 21:18:43,321 INFO common.Util: dfs.datanode.fileio.profiling.sampling.percentage set to 0. Disabling file IO profiling
2023-09-12 21:18:43,593 INFO blockmanagement.DatanodeManager: dfs.block.invalidate.limit : configured=1000, counted=60, effected=1000
2023-09-12 21:18:43,598 INFO blockmanagement.DatanodeManager: dfs.namenode.datanode.registration.ip-hostname-check=true
2023-09-12 21:18:43,613 INFO blockmanagement.BlockManager: dfs.namenode.startup.delay.block.deletion.sec is set to 000:00:00:00.000
2023-09-12 21:18:43,618 INFO blockmanagement.BlockManager: The block deletion will start around 2023 Sep 12 21:18:43
2023-09-12 21:18:43,625 INFO util.GSet: Computing capacity for map BlocksMap
2023-09-12 21:18:43,628 INFO util.GSet: VM type       = 64-bit
2023-09-12 21:18:43,637 INFO util.GSet: 2.0% max memory 475.6 MB = 9.5 MB
2023-09-12 21:18:43,638 INFO util.GSet: capacity      = 2^20 = 1048576 entries
2023-09-12 21:18:43,665 INFO blockmanagement.BlockManager: Storage policy satisfier is disabled
2023-09-12 21:18:43,668 INFO blockmanagement.BlockManager: dfs.block.access.token.enable = false
2023-09-12 21:18:43,702 INFO blockmanagement.BlockManagerSafeMode: dfs.namenode.safemode.threshold-pct = 0.999
2023-09-12 21:18:43,709 INFO blockmanagement.BlockManagerSafeMode: dfs.namenode.safemode.min.datanodes = 0
2023-09-12 21:18:43,711 INFO blockmanagement.BlockManagerSafeMode: dfs.namenode.safemode.extension = 30000
2023-09-12 21:18:43,717 INFO blockmanagement.BlockManager: defaultReplication         = 3
2023-09-12 21:18:43,722 INFO blockmanagement.BlockManager: maxReplication             = 512
2023-09-12 21:18:43,723 INFO blockmanagement.BlockManager: minReplication             = 1
2023-09-12 21:18:43,724 INFO blockmanagement.BlockManager: maxReplicationStreams      = 2
2023-09-12 21:18:43,726 INFO blockmanagement.BlockManager: redundancyRecheckInterval  = 3000ms
2023-09-12 21:18:43,727 INFO blockmanagement.BlockManager: encryptDataTransfer        = false
2023-09-12 21:18:43,730 INFO blockmanagement.BlockManager: maxNumBlocksToLog          = 1000
2023-09-12 21:18:43,923 INFO namenode.FSDirectory: GLOBAL serial map: bits=29 maxEntries=536870911
2023-09-12 21:18:43,927 INFO namenode.FSDirectory: USER serial map: bits=24 maxEntries=16777215
2023-09-12 21:18:43,928 INFO namenode.FSDirectory: GROUP serial map: bits=24 maxEntries=16777215
2023-09-12 21:18:43,931 INFO namenode.FSDirectory: XATTR serial map: bits=24 maxEntries=16777215
2023-09-12 21:18:43,964 INFO util.GSet: Computing capacity for map INodeMap
2023-09-12 21:18:43,967 INFO util.GSet: VM type       = 64-bit
2023-09-12 21:18:43,973 INFO util.GSet: 1.0% max memory 475.6 MB = 4.8 MB
2023-09-12 21:18:43,974 INFO util.GSet: capacity      = 2^19 = 524288 entries
2023-09-12 21:18:43,979 INFO namenode.FSDirectory: ACLs enabled? true
2023-09-12 21:18:43,980 INFO namenode.FSDirectory: POSIX ACL inheritance enabled? true
2023-09-12 21:18:43,980 INFO namenode.FSDirectory: XAttrs enabled? true
2023-09-12 21:18:43,987 INFO namenode.NameNode: Caching file names occurring more than 10 times
2023-09-12 21:18:43,996 INFO snapshot.SnapshotManager: Loaded config captureOpenFiles: false, skipCaptureAccessTimeOnlyChange: false, snapshotDiffAllowSnapRootDescendant: true, maxSnapshotLimit: 65536
2023-09-12 21:18:44,006 INFO snapshot.SnapshotManager: SkipList is disabled
2023-09-12 21:18:44,016 INFO util.GSet: Computing capacity for map cachedBlocks
2023-09-12 21:18:44,018 INFO util.GSet: VM type       = 64-bit
2023-09-12 21:18:44,022 INFO util.GSet: 0.25% max memory 475.6 MB = 1.2 MB
2023-09-12 21:18:44,034 INFO util.GSet: capacity      = 2^17 = 131072 entries
2023-09-12 21:18:44,070 INFO metrics.TopMetrics: NNTop conf: dfs.namenode.top.window.num.buckets = 10
2023-09-12 21:18:44,076 INFO metrics.TopMetrics: NNTop conf: dfs.namenode.top.num.users = 10
2023-09-12 21:18:44,082 INFO metrics.TopMetrics: NNTop conf: dfs.namenode.top.windows.minutes = 1,5,25
2023-09-12 21:18:44,141 INFO namenode.FSNamesystem: Retry cache on namenode is enabled
2023-09-12 21:18:44,143 INFO namenode.FSNamesystem: Retry cache will use 0.03 of total heap and retry cache entry expiry time is 600000 millis
2023-09-12 21:18:44,156 INFO util.GSet: Computing capacity for map NameNodeRetryCache
2023-09-12 21:18:44,157 INFO util.GSet: VM type       = 64-bit
2023-09-12 21:18:44,159 INFO util.GSet: 0.029999999329447746% max memory 475.6 MB = 146.1 KB
2023-09-12 21:18:44,159 INFO util.GSet: capacity      = 2^14 = 16384 entries
2023-09-12 21:18:44,234 INFO namenode.FSImage: Allocated new BlockPoolId: BP-1199298961-192.168.50.201-1694524724216
2023-09-12 21:18:44,315 INFO common.Storage: Storage directory /export/data/hadoop/dfs/name has been successfully formatted.
2023-09-12 21:18:44,450 INFO namenode.FSImageFormatProtobuf: Saving image file /export/data/hadoop/dfs/name/current/fsimage.ckpt_0000000000000000000 using no compression
2023-09-12 21:18:44,588 INFO namenode.FSImageFormatProtobuf: Image file /export/data/hadoop/dfs/name/current/fsimage.ckpt_0000000000000000000 of size 396 bytes saved in 0 seconds .
2023-09-12 21:18:44,615 INFO namenode.NNStorageRetentionManager: Going to retain 1 images with txid >= 0
2023-09-12 21:18:44,656 INFO namenode.FSNamesystem: Stopping services started for active state
2023-09-12 21:18:44,660 INFO namenode.FSNamesystem: Stopping services started for standby state
2023-09-12 21:18:44,672 INFO namenode.FSImage: FSImageSaver clean checkpoint: txid=0 when meet shutdown.
2023-09-12 21:18:44,677 INFO namenode.NameNode: SHUTDOWN_MSG: 
/************************************************************
SHUTDOWN_MSG: Shutting down NameNode at node1.node1.com/192.168.50.201
************************************************************/

Hadoop集群启动关闭-手动逐个进程启停

每台机器上每次手动启动关闭一个角色进程

  • HDFS集群
hdfs --daemon start namenode|datanode|secondarynamenode
hdfs --daemon stop namenode|datanode|secondarynamenode
  • YARN集群
yarn --daemon start resourcemanager|nodemanager
yarn --daemon stop resourcemanager|nodemanager

在node1中启动

hdfs --daemon start namenode
hdfs --daemon start datanode
yarn --daemon start resourcemanager
yarn --daemon start nodemanager
jps

在node2中启动

hdfs --daemon start datanode
hdfs --daemon start secondarynamenode
yarn --daemon start nodemanager
jps

在node3中启动

hdfs --daemon start datanode
yarn --daemon start nodemanager
jps

Hadoop集群启动关闭-shell脚本一键启停

在node1上，使用自带的shell脚本一键启动

前提：配置好机器之间的SSH免密登录和workers文件。

  • HDFS集群

start-dfs.sh

stop-dfs.sh

  • YARN集群

start-yarn.sh

stop-yarn.sh

  • Hadoop集群

start-all.sh
stop-all.sh

Hadoop Web UI页面-HDFS集群

地址：http://node1:9870

Hadoop Web UI页面-YARN集群

地址：http://node1:8088


http://www.ppmy.cn/news/1107293.html

相关文章

青大数据机构【2013】

青大数据机构【2013】

关键字&#xff1a; 邻接表空间复杂度、求无向图连通分量&#xff08;BFS、DFS&#xff09;、B树根节点最小关键字、平均查找长度最小的排序、二叉树排序叶子结点次序不变、不同次序建立二叉排序树及中序遍历、直接插入排序特点、强连通分量、邻接矩阵邻接表 一、单选&#x…
阅读更多...
cmd 90 validate error!（达梦数据库日志报错）

cmd 90 validate error!（达梦数据库日志报错）

达梦数据库报错 error-cmd 90 validate error! 环境介绍1 解决办法 环境介绍 某生产环境数据库启动后&#xff0c;dm_实例名_202309.log&#xff0c;偶尔报错cmd 90 validate error! 1 解决办法 接口用错了&#xff0c;消息非法&#xff0c;比如用 6 的 JDBC 连 7 或 7 的 …
阅读更多...
HTTP RESTFul RPC

HTTP RESTFul RPC

一、简介 &#xff08;1&#xff09;HTTP&#xff08;Hypertext Transfer Protocol&#xff09;是一种应用层协议。它经常用于在Web和服务器之间通讯&#xff0c;或服务与服务之间通讯。 &#xff08;2&#xff09;RESTFul 约束HTTP协议实现上的规范设计。 &#xff08;3&am…
阅读更多...
浑元太极大招“武德有悔”-UMLChina建模知识竞赛第4赛季第12轮

浑元太极大招“武德有悔”-UMLChina建模知识竞赛第4赛季第12轮

DDD领域驱动设计批评文集 “软件方法建模师”不再考查基础题 《软件方法》各章合集 参考潘加宇在《软件方法》和UMLChina公众号文章中发表的内容作答。在本文下留言回答。 最先答对所有3题者&#xff0c;即可获得本轮优胜。 所有题目的回答必须放在同一条消息中&#xff0…
阅读更多...
使用C语言EasyX 创建动态爱心背景

使用C语言EasyX 创建动态爱心背景

简介 在计算机图形学的世界中&#xff0c;有很多方法可以使程序的界面更加吸引人。在本篇博客中&#xff0c;我将向大家介绍如何使用 EasyX 图形库在 C 中创建一个动态的爱心背景。这不仅是一个简单的动画效果&#xff0c;它还包括背景的星星、旋转的心形以及一个美观的背景渐…
阅读更多...
橡胶密封件为什么会老化？

橡胶密封件为什么会老化？

橡胶密封件以其优良的密封性能被广泛应用于各个行业。然而&#xff0c;随着时间的推移&#xff0c;这些橡胶密封件往往会恶化和老化。在这篇文章中&#xff0c;我们将探讨橡胶密封件老化的原因。 1&#xff0c;导致橡胶密封件老化的主要因素之一是暴露在阳光和紫外线(UV)辐射下…
阅读更多...
AI项目六：WEB端部署YOLOv5

AI项目六：WEB端部署YOLOv5

若该文为原创文章&#xff0c;转载请注明原文出处。 一、介绍 最近接触网页大屏&#xff0c;所以就想把YOLOV5部署到WEB端&#xff0c;通过了解&#xff0c;知道了两个方法&#xff1a; 1、基于Flask部署YOLOv5目标检测模型。 2、基于Streamlit部署YOLOv5目标检测。 代码在…
阅读更多...
人工智能训练？量子计算机？显卡！【为什么人工智能需要强大的显卡而不是处理器？量子计算机可以吗？】

人工智能训练？量子计算机？显卡！【为什么人工智能需要强大的显卡而不是处理器？量子计算机可以吗？】

人工智能硬件与量子计算机&#xff1a;未来的竞争与合作 人工智能&#xff08;AI&#xff09;领域的发展一直以来都与硬件技术密不可分。随着深度学习和神经网络的崛起&#xff0c;图形处理单元&#xff08;GPU&#xff09;等硬件成为了AI训练的主要工具。然而&#xff0c;量子…
阅读更多...
最新文章

Copyright @ 2022~2023