cdh安装

1.         挂载系统ISO,获取linux系统文件及安装包

[root@tanj2 home]# chkconfig --level 2345 iptables off

[root@tanj2 home]# mkdir /mnt/cdrom

[root@tanj2 cdrom]# mount -o loop /home/CentOS-6.7-x86_64-bin-DVD1.iso /mnt/cdrom

[root@tanj2 mnt]# cp -r cdrom/  /home/os

2.         配置本地yum库

[root@tanj2 os]# cd /etc/yum.repos.d/

[root@tanj2 yum.repos.d]# vim os.repo

[os.repo]

name=os.repo

baseurl=file:///home/os

enabled=1

gpgcheck=0

3.         安装httpd服务

[root@tanj2 yum.repos.d]# yum install httpd

[root@tanj2 yum.repos.d]# service iptables stop

[root@tanj2 yum.repos.d]# service httpd start

[root@tanj2 yum.repos.d]# chkconfig --level 2345 httpd on

[root@tanj2 home]# cp -r os/ /var/www/html/

4.         数据硬盘分区以及挂载

[root@tanj2 home]# yum install parted

DEVICE_LIST="/dev/sda /dev/sdl /dev/sdg /dev/sdc /dev/sda /dev/sdd /dev/sdf /dev/sde /dev/sdj /dev/sdi /dev/sdh"

for DEVICE in $DEVICE_LIST

do

  echo "+++++create partition for $DEVICE......"

  parted -s $DEVICE mklabel gpt mkpart primary ext4 0% 100%

  PARTITION="$DEVICE""1"

  echo "+++++formatting $PARTITION......"

  mkfs.ext4 -T largefile $PARTITION

  #mkfs -t ext4 -j -m 1 -O extent,dir_index,spare_super $PARTITION

Done

[root@tanj2 home]# mkdir /data1

[root@tanj2 home]# vim /etc/fstab

/dev/sda1               /data1                  ext4    defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100        0 0

/dev/sdb1               /data2                  ext4    defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100        0 0

/dev/sdc1               /data3                  ext4    defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100        0 0

/dev/sdd1               /data4                  ext4    defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100        0 0

/dev/sde1               /data5                  ext4    defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100        0 0

/dev/sdf1               /data6                  ext4    defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100        0 0

/dev/sdg1               /data7                  ext4    defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100        0 0

/dev/sdh1               /data8                  ext4    defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100        0 0

/dev/sdi1               /data9                  ext4    defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100        0 0

/dev/sdj1               /data10                  ext4    defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100        0 0

/dev/sdl1               /data11                  ext4    defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100        0 0

5.         安装配置mariadb

[root@tanj8 yum.repos.d]# yum install MariaDB-server MariaDB-client

[root@tanj2 repodata]# service mysql start

[root@tanj2 ~]# vim /etc/my.cnf.d/server.cnf

[mysqld]

lower_case_table_names=1

character_set_server = utf8

[root@tanj2 ~]# mysql –uroot

GRANT ALL PRIVILEGES ON *.* TO 'root'@'%'IDENTIFIED BY '123456' WITH GRANT OPTION;

GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost'IDENTIFIED BY '123456' WITH GRANT OPTION;

flush privileges;

--hive数据库,为何使用latin1

create database hive DEFAULT CHARSET latin1;

--集群监控数据库

create database amon DEFAULT CHARSET utf8;

--hue数据库

create database hue DEFAULT CHARSET utf8;

--oozie数据库

create database oozie default charset utf8;

--受权

GRANT ALL PRIVILEGES ON *.* TO 'cmf'@'%'IDENTIFIED BY '123456' WITH GRANT OPTION;

flush privileges;

6.         关闭selinux

[root@tanj8 ~]# sestatus –v

[root@tanj8 yum.repos.d]# vim /etc/selinux/config

SELINUX=disabled

# SELINUXTYPE= can take one of these two values:

#     targeted - Targeted processes are protected,

#     mls - Multi Level Security protection.

SELINUXTYPE=targeted

7.         开启ntpd授时服务

[root@tanj8 ~]# yum install ntp

[root@tanj8 ~]# vim /etc/ntp.conf

server 10.0.61.4 prefer iburst

server 127.127.1.0

[root@tanj8 ~]# chkconfig --level 2345 ntpd on

[root@tanj8 ~]# service ntpd start

8.         系统优化

[root@tanj1 ~]# vim /etc/rc.local

echo never > /sys/kernel/mm/transparent_hugepage/defrag

echo never > /sys/kernel/mm/transparent_hugepage/enabled

[root@tanj2 etc]# echo never > /sys/kernel/mm/transparent_hugepage/defrag

[root@tanj2 etc]# echo never > /sys/kernel/mm/transparent_hugepage/enabled

[root@tanj2 etc]# vim /etc/sysctl.conf

vm.swappiness=10

[root@2 etc]# sysctl -p

9.         ssh 免密码登陆（将A机器生成公钥追加到B机器~/.ssh/authorized_keys文件中，那么A就能免密登陆B机器）

[root@tanj2 .ssh]# ssh-keygen -t rsa

[root@tanj2 .ssh]# cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

[root@tanj3 .ssh]# scp authorized_keys root@B:/root/.ssh/

10.     设置hostname(根据实际ip映射)

[root@tanj3 .ssh]# vim /etc/hosts

127.0.0.1 tanj1

127.0.0.1 tanj2

127.0.0.1 tanj3

11.     卸载jdk8

[root@tanj2 ~]# rpm -qa|grep jdk

[root@tanj2 ~]# rpm -e  --nodeps  java-1.8.0-openjdk-1.8.0.45-35.b13.el6.x86_64

[root@tanj2 ~]# rpm -e  --nodeps  java-1.8.0-openjdk-headless-1.8.0.45-35.b13.el6.x86_64

12.     安装Oracle JDK和Cloudera Manager以及Cloudera Manager Agent

[root@tanj2 cm511]# yum install oracle-j2sdk1.7

#集群上主节点安装manager-server manager-daemons

[root@tanj2 local]# yum install cloudera-manager-daemons cloudera-manager-server

#集群上全部的节点包括主节点都须要安装CM Agent

[root@tanj2 local]#yum install cloudera-manager-agent cloudera-manager-daemons

[root@tanj2 local]# yum list installed

[root@tanj2 local]# yum list installed|grep oracle

[root@tanj2 local]# rpm -ql oracle-j2sdk1.7.x86_64

#集群的全部节点上设置/etc/cloudera-scm-agent/config.ini文件中server-host和server-port两个属性，是CM Agent链接的CM Server进程所运行的服务器IP和端口

[root@tanj2 local]# vim /etc/cloudera-scm-agent/config.ini

[General]

# Hostname of the CM server.

server_host=cre2

# Port that the CM server is listening on.

server_port=7182

[root@tanj2 local]# chkconfig --level 2345 cloudera-scm-agent on

[root@tanj2 local]# chkconfig --level 2345 cloudera-scm-server on

[root@tanj2 local]# sestatus –v

[root@tanj2 java]# ln -s mysql-connector-java.jar /usr/share/java/mysql-connector-java-5.1.42-bin.jar

[root@tanj2 local]# service cloudera-scm-agent start

[root@tanj2 schema]# /usr/share/cmf/schema/scm_prepare_database.sh mysql -uroot -p123456 --scm-host localhost cmf cmf 123456

[root@tanj2 cloudera-scm-server]# vim /etc/cloudera-scm-server/db.properties

com.cloudera.cmf.db.type=mysql

com.cloudera.cmf.db.host=localhost

com.cloudera.cmf.db.name=cmf

com.cloudera.cmf.db.user=cmf

com.cloudera.cmf.db.setupType=EXTERNAL

com.cloudera.cmf.db.password=123456

[root@tanj2 java]# service cloudera-scm-server start

[root@tanj2 java]# less /var/log/cloudera-scm-server/cloudera-scm-server.log

http://127.0.0.1:7180

13.     cdh安装

14.     安装hdfs优化

[root@tanj2 ~]# sudo -uhdfs hdfs dfsadmin -safemode leave

15.     安装hive

[root@tanj2 ~]# sudo -uhdfs hdfs dfsadmin -safemode leave