1、常规初始化操作
三个节点都需要执行
hostnamectl set-hostname cdhmaster
 yum -y install vim lrzsz net-tools
cat >>/etc/hosts<<EOF
192.168.180.210 cdhmaster
 192.168.180.200 cdhslave01
 192.168.180.190 cdhslave02
 EOF
systemctl stop firewalld.service && systemctl disable firewalld.service
sed -i ‘/^SELINUX=/s/enforcing/disabled/’ /etc/selinux/config && setenforce 0
2、配置时间同步
cdhmaster:
yum install chrony -y
vim /etc/chrony.conf
server ntp6.aliyun.com iburst
 allow 192.168.0.0/16
systemctl enable chronyd.service && systemctl restart chronyd.service
chronyc sources && chronyc -a makestep
cdhslave01/cdhslave02:
yum install chrony -y
vim /etc/chrony.conf
 server cdhmaster iburst
 systemctl enable chronyd.service && systemctl restart chronyd.service && chronyc sources
3、配置本机SSH免密码登录
ssh-keygen -t rsa
ssh-copy-id cdhmaster
ssh-copy-id cdhslave01
ssh-copy-id cdhslave02
4、安装jdk
 三个节点都要执行
tar zxf jdk-8u171-linux-x64.tar.gz -C /opt/
mv /opt/jdk1.8.0_171/ /opt/jdk
cat >>/etc/profile<<EOF
 export JAVA_HOME=/opt/jdk
 export PATH= J A V A H O M E / b i n : JAVA_HOME/bin: JAVAHOME/bin:PATH
 EOF
 source /etc/profile && java -version
5、禁用交换分区和透明大页
 三个节点都要操作
sysctl -w vm.swappiness=0
echo “vm.swappiness=0” >> /etc/sysctl.conf
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo “echo never > /sys/kernel/mm/transparent_hugepage/defrag” >> /etc/rc.d/rc.local
echo “echo never > /sys/kernel/mm/transparent_hugepage/enabled” >> /etc/rc.d/rc.local
chmod +x /etc/rc.d/rc.local
6、安装数据库
 只在主节点cdhmastar上操作
rpm -qa | grep mariadb
rpm -e --nodeps mariadb-libs-5.5.68-1.el7.x86_64
yum install -y libaio
 useradd mysql -s /sbin/nologin
tar zvxf mysql-5.7.22-linux-glibc2.12-x86_64.tar.gz
mv mysql-5.7.22-linux-glibc2.12-x86_64 /usr/local/mysql
cd /usr/local/mysql/bin
./mysqld --initialize --user=mysql --datadir=/usr/local/mysql/data
自动生成数据库密码 IrszF>8vd(gy
cp /usr/local/mysql/support-files/mysql.server /etc/init.d/mysqld
vim /etc/my.cnf
[client]
 port = 3306
 socket = /usr/local/mysql/data/mysql.sock
 [mysqld]
 port = 3306
 socket = /usr/local/mysql/data/mysql.sock
 pid_file = /usr/local/mysql/data/mysql.pid
 datadir = /usr/local/mysql/data
 default_storage_engine = InnoDB
 max_allowed_packet = 512M
 max_connections = 2048
 open_files_limit = 65535
 skip-name-resolve
 lower_case_table_names=1
 character-set-server = utf8mb4
 collation-server = utf8mb4_unicode_ci
 init_connect=‘SET NAMES utf8mb4’
 innodb_buffer_pool_size = 1024M
 innodb_log_file_size = 2048M
 innodb_file_per_table = 1
 innodb_flush_log_at_trx_commit = 0
 key_buffer_size = 64M
 log-error = /usr/local/mysql/data/mysql_error.log
 log-bin = /usr/local/mysql/data/mysql-bin
 slow_query_log = 1
 slow_query_log_file = /usr/local/mysql/data/mysql_slow_query.log
 long_query_time = 5
 tmp_table_size = 32M
 max_heap_table_size = 32M
 query_cache_type = 0
 query_cache_size = 0
 server-id=1
/etc/init.d/mysqld start
ln -s /usr/local/mysql/bin/* /usr/sbin/
mysqladmin -uroot -p password ‘123456’
mysql -u root -p123456
alter user ‘root’@‘localhost’;
alter user ‘root’@‘localhost’ identified by ‘123456’;
create database hive default charset utf8 collate utf8_general_ci;
create database oozie default charset utf8 collate utf8_general_ci;
grant all privileges on . to ‘cdh’@‘localhost’ identified by ‘123456’ with grant option;
grant all privileges on . to ‘cdh’@‘%’ identified by ‘123456’ with grant option;
grant all privileges on . to ‘root’@‘127.0.0.1’ identified by ‘123456’ with grant option;
grant all privileges on . to ‘scm’@‘127.0.0.1’ identified by ‘123456’ with grant option;
flush privileges;
7、安装CDH
 所有节点上安装依赖包
yum install -y psmisc libxslt libxslt-python perl
上传cloudera-manager-centos7-cm5.14.0_x86_64.tar.gz、mysql-connector-java-5.1.46.tar.gz到服务器
tar zxf cloudera-manager-centos7-cm5.14.0_x86_64.tar.gz -C /opt/
mv /opt/cm-5.14.0/ /opt/cm
tar zxf mysql-connector-java-5.1.46.tar.gz
cd mysql-connector-java-5.1.46
chmod +x mysql-connector-java-5.1.46-bin.jar
cp mysql-connector-java-5.1.46-bin.jar /opt/cm/share/cmf/lib/mysql-connector-java.jar
mkdir /usr/share/java
cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar
在三台机器上创建cloudera-scm用户
useradd --system --home=/opt/cm/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment “Cloudera SCM User” cloudera-scm
相关选项说明
–no-create-home 相当于-M,不创建用户的主目录
–shell 用户登录的shell,/bin/false禁止用户登录
–system 创建一个系统用户,在sys_uid_max中取ID
–comment 添加备注
/opt/cm/share/cmf/schema/scm_prepare_database.sh mysql cm -h localhost -uroot -p’123456’ scm ‘123456’
 vim /opt/cm/etc/cloudera-scm-agent/config.ini
server_host=192.168.180.210
同步Cloudera Manager 数据到其他节点
scp -r /opt/cm cdhslave01:/opt/
scp -r /opt/cm cdhslave02:/opt/
上传manifest.json、CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel、CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha1文件到cdhmaster服务器上
mv manifest.json CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha1 /opt/cloudera/parcel-repo/
cd /opt/cloudera/parcel-repo/
mv CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha1 CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha
vim /opt/cm/etc/cloudera-scm-server/db.properties
com.cloudera.cmf.db.useSSL=true
mkdir /var/lib/cloudera-scm-server
/opt/cm/etc/init.d/cloudera-scm-server start
/opt/cm/etc/init.d/cloudera-scm-agent start
netstat -anpt | grep 7180
在slave上只需启动代理服务
/opt/cm/etc/init.d/cloudera-scm-agent start
netstat -anpt | grep -E ‘19001|9000’
8、安装配置CDH 群集
 访问CDH控制台
 http://192.168.180.210:7180 默认的用户名和密码分别是admin 和admin。
三台机器都执行
 上传mysql-connector-java-5.1.46.tar.gz到服务器
cd mysql-connector-java-5.1.46/
cp mysql-connector-java-5.1.46-bin.jar /opt/cloudera/parcels/CDH/lib/hive/lib/mysql-connector-java.jar
cp mysql-connector-java-5.1.46-bin.jar /opt/cloudera/parcels/CDH/lib/hadoop/lib/mysql-connector-java.jar
cp mysql-connector-java-5.1.46-bin.jar /var/lib/oozie/mysql-connector-java.jar
将上述三个文件复制到两台从节点
scp mysql-connector-java-5.1.46-bin.jar cdhslave01:/opt/cloudera/parcels/CDH/lib/hadoop/lib/mysql-connector-java.jar
scp mysql-connector-java-5.1.46-bin.jar cdhslave01:/opt/cloudera/parcels/CDH/lib/hive/lib/mysql-connector-java.jar
scp mysql-connector-java-5.1.46-bin.jar cdhslave01:/var/lib/oozie/mysql-connector-java.jar
scp mysql-connector-java-5.1.46-bin.jar cdhslave02:/opt/cloudera/parcels/CDH/lib/hive/lib/mysql-connector-java.jar
scp mysql-connector-java-5.1.46-bin.jar cdhslave02:/opt/cloudera/parcels/CDH/lib/hadoop/lib/mysql-connector-java.jar
scp mysql-connector-java-5.1.46-bin.jar cdhslave02:/var/lib/oozie/mysql-connector-java.jar
9、配置并添加Kafka 服务
先激活kafka
上传软件
mv KAFKA-1.2.0.jar /opt/cloudera/csd/
mv KAFKA-2.0.2-1.2.0.2.p0.5-el7.parcel /opt/cloudera/parcel-repo/
mv KAFKA-2.0.2-1.2.0.2.p0.5-el7.parcel.sha1 /opt/cloudera/parcel-repo/KAFKA-2.0.2-1.2.0.2.p0.5-el7.parcel.sha