CDH安装教程

安装依赖

yum -y install gcc python-devel cyrus-sasl* chrony iotop yum-utils yum-plugin-downloadonly psmisc ntp ntpdate httpd createrepo wget git rsync mod_ssl cyrus-sasl-plain cyrus-sasl-devel cyrus-sasl-gssapi libxslt krb5-devel cyrus-sasl-gssapi cyrus-sasl-devel libxml2-devel libxslt-devel openldap-devel python-devel python-simplejson sqlite-devel
# 删除mariadb
yum -y remove mariadb-libs

systemctl start ntpd
systemctl enable ntpd

修改一些参数

sysctl vm.swappiness=10

echo 'vm.swappiness=10'>> /etc/sysctl.conf
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' >> /etc/rc.local
echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >> /etc/rc.local

安装jdk

全部节点都需要安装

vim	 /etc/profile

在文件末尾添加

export JAVA_HOME=/usr/local/jdk/jdk1.8.0_202
export PATH=$PATH:$JAVA_HOME/bin

使修改生效

 source /etc/profile

查看java是否安装完成

 javac -version 
 jps

安装MySQL

选择node1

#下载这6个MySQL的rpm包
https://downloads.mysql.com/archives/get/p/23/file/mysql-community-common-5.7.17-1.el7.x86_64.rpm
https://downloads.mysql.com/archives/get/p/23/file/mysql-community-libs- 5.7.17-1.el7.x86_64.rpm
https://downloads.mysql.com/archives/get/p/23/file/mysql-community-libs-c ompat-5.7.17-1.el7.x86_64.rpm
https://downloads.mysql.com/archives/get/p/23/file/mysql-community-devel- 5.7.17-1.el7.x86_64.rpm
https://downloads.mysql.com/archives/get/p/23/file/mysql-community-client -5.7.17-1.el7.x86_64.rpm
https://downloads.mysql.com/archives/get/p/23/file/mysql-community-server -5.7.17-1.el7.x86_64.rpm

#安装
yum -y localinstall mysql-community-common-5.7.19-1.el7.x86_64.rpm 
yum -y localinstall mysql-community-libs-5.7.19-1.el7.x86_64.rpm 
yum -y localinstall mysql-community-libs-compat-5.7.19-1.el7.x86_64.rpm 
yum -y localinstall mysql-community-devel-5.7.19-1.el7.x86_64.rpm 
yum -y localinstall mysql-community-client-5.7.19-1.el7.x86_64.rpm 
yum -y localinstall mysql-community-server-5.7.19-1.el7.x86_64.rpm

#启动
systemctl start mysqld

#开机自启动
systemctl enable mysqld

#查看临时密码
cat /var/log/mysqld.log | grep password

mysql -u root -p 
#首次登录设置密码
SET PASSWORD = PASSWORD('Zykj@123456');
quit;
#登陆设置允许外网访问当前的MySQL
mysql -u root -p 'Zykj@123456'
use mysql;
update user set host='%' where host='localhost';
delete from user where host !='%';
flush privileges;
quit;
mysql -u root -p 'Zykj@123456'
#创建ClouderaManager需要用到的几个数据库
create database cmf default character set='utf8';
create database amon default character set='utf8';
create database hive default character set='latin1';
create database hue default character set='utf8';
create database oozie default character set='utf8';
quit;

#整理mysql的驱动包,全部节点
mv mysql-connector-java-5.1.39.jar  mysql-connector-java.jar
mkdir /usr/share/java
mv mysql-connector-java.jar /usr/share/java/

下载CM组件

https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/RPMS/x86_64/cloudera-m anager-daemons-6.2.1-1426065.el7.x86_64.rpm
https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/RPMS/x86_64/cloudera-m anager-agent-6.2.1-1426065.el7.x86_64.rpm
https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/RPMS/x86_64/cloudera-m anager-server-6.2.1-1426065.el7.x86_64.rpm

#全部节点都安装daemons和agent
yum -y localinstall cloudera-manager-daemons-6.2.1-1426065.el7.x86_64.rpm
yum -y localinstall cloudera-manager-agent-6.2.1-1426065.el7.x86_64.rpm

#cdh001上安装server
yum -y localinstall cloudera-manager-server-6.2.1-1426065.el7.x86_64.rpm 
#到此为止，我们的CM服务安装完毕

下载CDH的parcel包

这些都在node1上操作

https://archive.cloudera.com/cdh6/6.2.1/parcels/CDH-6.2.1-1.cdh6.2.1.p0.1 425774-el7.parcel
https://archive.cloudera.com/cdh6/6.2.1/parcels/CDH-6.2.1-1.cdh6.2.1.p0.1 425774-el7.parcel.sha1
https://archive.cloudera.com/cdh6/6.2.1/parcels/manifest.json
#修改server和agent的配置文件
https://archive.cloudera.com/phoenix/6.2.0/parcels/PHOENIX-5.0.0-cdh6.2. 0.p0.1308267-el7.parcel.sha
https://archive.cloudera.com/phoenix/6.2.0/parcels/PHOENIX-5.0.0-cdh6.2. 0.p0.1308267-el7.parcel
https://archive.cloudera.com/phoenix/6.2.0/csd/PHOENIX-1.0.jar

#然后把PHOENIX-1.0.jar放到/opt/cloudera/csd目录 
[root@cdh001csd]#pwd
/opt/cloudera/csd
[root@cdh001csd]#ll
total8 
-rw-r--r--1 root root 5306 May 6 13:41 PHOENIX-1.0.jar
#把凡是带sha1的都改成sha 
#把上面下载的包都放在/opt/cloudera/parcel-repo下
[root@cdh001parcel-repo]#pwd
/opt/cloudera/parcel-repo
[root@cdh001parcel-repo]#ll
total 2522808
-rw-r--r--1 root root 2093332003May213:39CDH-6.2.1-1.cdh6.2.1.p0.14 25774-el7.parcel
-rw-r--r--1 root root 40May613:39CDH-6.2.1-1.cdh6.2.1.p0.1425774-el 7.parcel.sha
-rw-r--r--1 root root 33875May613:39manifest.json
-rw-r--r--1 root root 402216960May613:42PHOENIX-5.0.0-cdh6.2.0.p0.1308267-el7.parcel
-rw-r--r--1 root root 41May613:41PHOENIX-5.0.0-cdh6.2.0.p0.1308267- el7.parcel.sha

修改server和agent的配置文件

#node1上修改server
vim /etc/cloudera-scm-server/db.properties
#12行去掉注释把localhost改成node1
#15行去掉注释
#18行去掉注释，把cmf改成root
#21行去掉注释，密码那里填写Zykj@123456

# 全部节点修改agent
vim /etc/cloudera-scm-agent/config.ini
#15行把localhost修改为node1

修改SELINUX

SELinux一共有3种状态，分别是Enforcing，Permissive和Disabled状态。

Enforcing：强制模式。代表SELinux在运行中，且已经开始限制domain/type之间的验证关系

Permissive：宽容模式。代表SELinux在运行中，不过不会限制domain/type之间的验证关系，即使验证不正确，进程仍可以对文件进行操作。不过如果验证不正确会发出警告

Disabled：关闭模式。SELinux并没有实际运行

可以通过setenforce命令来设置前面两种状态，而如果想修改为disable状态，需要修改配置文件，同时重启系统。

Enforcing状态的值是1，permissive状态是0，因此设置为permissive，命令就是：setenforce 0|1

临时关闭SELinux：setenforce 0

临时开启SELinux：setenforce 1

查看当前状态命令：getenforce

vim /etc/selinux/config
# 修改
 SELINUX=disabled

关闭防火墙

  systemctl stop firewalld

  systemctl disable firewalld

启动CM，开始安装CDH集群

# 在node1上启动server
systemctl start cloudera-scm-server
# 另开一个窗口，查看相关日志。有异常就解决异常
tail -200f /var/log/cloudera-scm-server/cloudera-scm-server.log
# 这个异常可以忽略 ERROR ParcelUpdateService:com.cloudera.parcel.component s.ParcelDownloaderImpl: Unable to retrieve remote parcel repository manifes t

# 在全部节点上启动agent
systemctl start cloudera-scm-agent

# 当在 node1上 netstat -tunlp | grep 7180 有内容时，说明我们可以访问web页 了

#重启命令
systemctl restart cloudera-scm-server
systemctl restart cloudera-scm-agent

报错

执行systemctl start cloudera-scm-server命令报如下错误：

[root@s30 ~]# systemctl status cloudera-scm-server
● cloudera-scm-server.service - Cloudera CM Server Service
   Loaded: loaded (/usr/lib/systemd/system/cloudera-scm-server.service; enabled; vendor preset: disabled)
   Active: failed (Result: start-limit) since 五 2020-08-07 10:01:06 CST; 7s ago
  Process: 4537 ExecStart=/opt/cloudera/cm/bin/cm-server (code=exited, status=1/FAILURE)
  Process: 4533 ExecStartPre=/opt/cloudera/cm/bin/cm-server-pre (code=exited, status=0/SUCCESS)
 Main PID: 4537 (code=exited, status=1/FAILURE)

8月 07 10:01:06 s30 systemd[1]: cloudera-scm-server.service: main process exited, code=exited, status=1/FAILURE
8月 07 10:01:06 s30 systemd[1]: Unit cloudera-scm-server.service entered failed state.
8月 07 10:01:06 s30 systemd[1]: cloudera-scm-server.service failed.
8月 07 10:01:06 s30 systemd[1]: cloudera-scm-server.service holdoff time over, scheduling restart.
8月 07 10:01:06 s30 systemd[1]: Stopped Cloudera CM Server Service.
8月 07 10:01:06 s30 systemd[1]: start request repeated too quickly for cloudera-scm-server.service
8月 07 10:01:06 s30 systemd[1]: Failed to start Cloudera CM Server Service.
8月 07 10:01:06 s30 systemd[1]: Unit cloudera-scm-server.service entered failed state.
8月 07 10:01:06 s30 systemd[1]: cloudera-scm-server.service failed.

这错误是找不到Java环境

原来程序默认会去使用/usr/java下的jdk，所以解决办法执行以下两条命令即可：

mkdir -p /usr/java
ln -s /usr/local/java/jdk1.8.0_202  /usr/java/

参考 (opens new window)

WEB网页配置

给集群起个名字

以安装zookeeper为例

测试部分

1、Kafka测试 2 创建一个topic
kafka-topics --zookeeper node1:2181,node2:2181,node3:2181 --create--r eplication-factor 3 --partitions 10 --topic t1

开启kafka控制台消费者
kafka-console-consumer --bootstrap-server cdh003:9092,cdh004:9092,cdh005: 9092 --topic t1

去别的节点开启一个Kafka控制台生产者
kafka-console-producer --broker-list cdh003:9092,cdh004:9092,cdh005:9092 --topic t1
一边消费数据一边写入数据 

2、Hive测试
通过Hive建立一张表
create table if not exists t1(idint,namestring,scoredouble)row format delimited fields terminated by ',' stored as parquet; 
写个带有聚合函数的让它跑一下MapReduce
select count(*) from t1; 
3、Phoenix测试
create table if not exists t1(
id Integer,
name varchar,
clazz varchar,
score double
constraint pk primary key(id)
);
select * from t1;
upsert into t1 values(1001,'zhangsan','606',87.5);
select count(*) from t1;
select * from t1;

4、Impala测试
create table t2
(
name string,
dt string,
primary key(name,dt)
)
partition by hash partitions 16
stored as kudu tblproperties(
 'kudu.master_addresses'='cdh001:7051'
 );

select * from t2;
insert into t2 values('zhangsan','2020-05-06');
select count(*) from t2;
select * from t2;

问题总结

CDH安装失败，重装操作

删除Agent节点的UUID

rm -rf /var/lib/cloudera-scm-agent/cm_guid

清空主节点CM数据库

--进入主节点的mysql数据库，然后删除数据库
drop database cmf;

删除Agent节点namenode和datanode节点信息

rm -rf /data1/dfs/nn/*
rm -rf /data1/dfs/fn/*
rm -rf /data1/dfs/dn/*

在主节点上重新初始化CM数据库

重新创建数据库 cmf

create database cmf default character set='utf8';

cd /var/lib/cloudera-scm-agent
./scm_prepare_database.sh -hnode1 -P3306 --scm-host node1 --force mysql cmf root Zykj@123456

执行启动脚本

# 主节点
systemctl start cloudera-scm-server
# Agent节点
systemctl start cloudera-scm-agent

通过主节点的7180端口重新进行CDH的安装 http://主节点IP:7180/cmf/login

主机上未检测到 Java JDK

CDH添加新主机 - 主机上未检测到 Java JDK

在cloudera manager管理界面上操作往CDH集群添加新主机时，跳过JDK安装的步骤，因为服务器一般都会安装有JDK

全部步骤完成后，到主机配置也发现由如下提示：“主机上未检测到 Java JDK。”

如果不解决这个问题，角色将会无法启动。

在配置页面搜索java，配置对应路径，保存配置即可