HadoopHadoop
2025-05-04 17:20阅读:34评论:0
Linux 伪分布式部署Hadoop-3.4.0
准备工作 下载 hadoop-3.4.0 下载jdk8 环境配置 免密码登陆-ssh密钥配置 hadoop-env.sh 配置 core-site.xml配置 yarn-site.xml 配置 mapred-site.xml 配置 hdfs-site.xml 配置 初始化数据源 启动服务 访问Had
_PROTECTED0__
- 下载 hadoop-3.4.0
- 下载jdk8
##### 环境配置
1
2
3
4
5
6
export JAVA_HOME=/usr/local/jdk8export PATH=$PATH:$JAVA_HOME/bin export HADOOP_HOME=/usr/local/hadoop-3.4.0 # 这是我部署Hadoop根目录export PATH=$HADOOP_HOME/bin:$PATH _PROTECTED0__
1
2
3
4
5
cd ~/.ssh/ # 若没有该目录，请先执行一次ssh localhostssh-keygen -t rsa # 会有提示，都按回车就可以cat ./id_rsa.pub >> ./authorized_keys # 加入授权ssh localhost # 验证 _PROTECTED0__
1
2
3
4
5
6
7
8
9
export JAVA_HOME=/usr/local/jdk8export HDFS_NAMENODE_USER=rootexport HDFS_DATANODE_USER=rootexport HDFS_SECONDARYNAMENODE_USER=rootexport YARN_RESOURCEMANAGER_USER=rootexport YARN_NODEMANAGER_USER=root#export HADOOP_PID_DIR=/usr/local/hadoop-3.4.0/pidsexport HADOOP_LOG_DIR=/usr/local/hadoop-3.4.0/logs _PROTECTED0__
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
<configuration> <property> <name>fs.defaultFS</name> <value>hdfs://10.211.55.7:9000</value> </property> <property> <name>hadoop.tmp.dir</name> <value>/usr/local/hadoop-3.4.0/tmp</value> </property> <property> <name>dfs.webhdfs.enabled</name> <value>true</value> </property> <property> <name>hadoop.proxyuser.root.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.root.groups</name> <value>*</value> </property></configuration> _PROTECTED0__
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
<configuration> <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.nodemanager.localizer.address</name> <value>0.0.0.0:8140</value> </property> <property> <name>yarn.resourcemanager.hostname</name> <value>10.211.55.7</value> </property> <property> <name>yarn.log-aggregation-enable</name> <value>true</value> </property> <property> <name>yarn.log-aggregation.retain-seconds</name> <value>604800</value> </property> <property> <name>yarn.log.server.url</name> <value>http://10.211.55.7:19888/jobhistory/logs</value> </property></configuration> _PROTECTED0__
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
<configuration> <property> <name>mapreduce.framework.name</name> <value>yarn</value> </property> <property> <name>yarn.app.mapreduce.am.env</name> <value>HADOOP_MAPRED_HOME=/usr/local/hadoop-3.4.0</value> </property> <property> <name>mapreduce.map.env</name> <value>HADOOP_MAPRED_HOME=/usr/localft/hadoop-3.4.0</value> </property> <property> <name>mapreduce.reduce.env</name> <value>HADOOP_MAPRED_HOME=/usr/local/hadoop-3.4.0</value> </property> <property> <name>mapreduce.jobhistory.address</name> <value>10.211.55.7:10020</value> </property> <property> <name>mapreduce.jobhistory.webapp.address</name> <value>10.211.55.7:19888</value> </property></configuration> _PROTECTED0__
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
<configuration> <property> <name>dfs.namenode.secondary.http-address</name> <value>10.211.55.7:50090</value> </property> <property> <name>dfs.replication</name> <value>1</value> </property> <property> <name>dfs.namenode.name.dir</name> <value>file:/usr/local/hadoop-3.4.0/hdfs/name</value> </property> <property> <name>dfs.datanode.data.dir</name> <value>file:/usr/local/hadoop-3.4.0/hdfs/data</value> </property> <property> <name>dfs.permissions.enabled</name> <value>false</value> </property> <property> <name>hadoop.proxyuser.root.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.root.groups</name> <value>*</value> </property></configuration> _PROTECTED0__
1
2
3
# 终端执行命令hadoop namenode -format _PROTECTED0__
1
2
3
4
5
6
7
# 进入目录hadoop-3.4.0/sbin#启动服务bash start-all.sh#关闭服务bash stop-all.sh _PROTECTED0__
- http://10.211.55.7:9870