Linux下安装配置Hadoop

背景

CentOS7
jdk1.8
hadoop-2.9.2

下载hadoop

wget http://mirrors.shu.edu.cn/apache/hadoop/common/hadoop-2.9.2/hadoop-2.9.2.tar.gzjava

tar -xzvf hadoop-2.9.2.tar.gznode

mv hadoop-2.9.2 /usr/local/hadoopweb

安装jdk8环境

参考Linux下安装配置JDKapache

vim /etc/profilevim

JAVA_HOME=/usr/local/jdk1.8.0_171
JRE_HOME=/usr/local/jdk1.8.0_171/jre
PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
export PATH

source /etc/profilecentos

echo $JAVA_HOME /usr/local/jdk1.8.0_171app

hadoop环境变量

vim /etc/profiledom

HADOOP_HOME=/usr/local/hadoop
PATH=$PATH:$HADOOP_HOME/bin
export PATH

source /etc/profilewebapp

hadoop versionoop

hadoop环境配置

hadoop 配制文件

hadoop/etc/hadoop/hadoop-env.sh 
hadoop/etc/hadoop/yarn-env.sh 
hadoop/etc/hadoop/core-site.xml 
hadoop/etc/hadoop/hdfs-site.xml 
hadoop/etc/hadoop/mapred-site.xml 
hadoop/etc/hadoop/yarn-site.xml

hadoop-env.sh

cd /usr/local/hadoop/etc/hadoop/

vim hadoop-env.sh

# The java implementation to use.
#export JAVA_HOME=${JAVA_HOME}
export JAVA_HOME=/usr/local/jdk1.8.0_171

yarn-env.sh

cd /usr/local/hadoop/etc/hadoop/

vim yarn-env.sh

export JAVA_HOME=/usr/local/jdk1.8.0_171

core-site.xml 

cd /usr/local/hadoop

mkdir tmp

cd /usr/local/hadoop/etc/hadoop/

vim core-site.xml 

<configuration>
    <property>
        <name>fs.default.name</name>
        <value>hdfs://192.168.140.134:9000</value>
        <description>HDFS的URI,文件系统://namenode标识:端口号</description>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/usr/local/hadoop/tmp</value>
        <description>namenode上本地的hadoop临时文件夹</description>
    </property>
</configuration>

--添加权限

hdfs dfs -chmod -R 777 /tmp

注意:这里的ip地址 192.168.140.134 须要根据须要修改;

hdfs-site.xml 

cd /usr/local/hadoop/etc/hadoop/

vim hdfs-site.xml 

<configuration>
    <!—hdfs-site.xml-->
    <property>
        <name>dfs.name.dir</name>
        <value>/data0/hadoop/hdfs/name</value>
        <description>namenode上存储hdfs名字空间元数据 </description> 
    </property>
    <property>
        <name>dfs.data.dir</name>
        <value>/data0/hadoop/hdfs/data</value>
        <description>datanode上数据块的物理存储位置</description>
    </property>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
        <description>副本个数,配置默认是3,应小于datanode机器数量</description>
    </property>
</configuration>
数据目录 不存在,记的建立

mkdir -p /data0/hadoop/hdfs/name

mkdir -p /data0/hadoop/hdfs/data

mapred-site.xml 

cd /usr/local/hadoop/etc/hadoop/

vim mapred-site.xml

<configuration>
    <property>
            <name>mapreduce.framework.name</name>
            <value>yarn</value>
    </property>
</configuration>

yarn-site.xml 

cd /usr/local/hadoop/etc/hadoop/

vim yarn-site.xml

<configuration>
    <property>
            <name>yarn.nodemanager.aux-services</name>
            <value>mapreduce_shuffle</value>
    </property>
    <property>
            <name>yarn.resourcemanager.webapp.address</name>
            <value>localhost:8099</value>
    </property>
</configuration>

Hadoop启动 

cd /usr/local/hadoop/

格式化namenode

./bin/hdfs namenode –format

start-all.sh 启动

cd /usr/local/hadoop

./sbin/start-all.sh

分批启动

  • 启动NameNode 和 DataNode 守护进程

./sbin/start-dfs.sh

启动日志

./sbin/start-dfs.sh
Starting namenodes on [localhost]
root@localhost's password: 
localhost: starting namenode, logging to /usr/local/hadoop/logs/hadoop-root-namenode-localhost.localdomain.out
root@localhost's password: 
localhost: starting datanode, logging to /usr/local/hadoop/logs/hadoop-root-datanode-localhost.localdomain.out
Starting secondary namenodes [0.0.0.0]
root@0.0.0.0's password: 
0.0.0.0: starting secondarynamenode, logging to /usr/local/hadoop/logs/hadoop-root-secondarynamenode-localhost.localdomain.out
  • 启动ResourceManager 和 NodeManager 守护进程

./sbin/start-yarn.sh

启动日志

./sbin/start-yarn.sh
starting yarn daemons
starting resourcemanager, logging to /usr/local/hadoop/logs/yarn-root-resourcemanager-localhost.localdomain.out
root@localhost's password: 
localhost: starting nodemanager, logging to /usr/local/hadoop/logs/yarn-root-nodemanager-localhost.localdomain.out

### 启动验证 

jps

6097 NodeManager
11044 Jps
7497 -- process information unavailable
8256 Worker
5999 ResourceManager
5122 SecondaryNameNode
8106 Master
4836 NameNode
4957 DataNode

centos7关闭防火墙

systemctl stop firewalld.service

相关文章
相关标签/搜索