Kafka部署

鸡汤: 而立之年，适当放弃一些，做减法运算，轻装上阵。

1. 环境¶

主机名	IP	要安装的软件
master	192.168.186.10	zookeeper，kafka
slave1	192.168.186.11	zookeeper，kafka
slave2	192.168.186.12	zookeeper，kafka

2. 部署¶

2.1 下载安装配置¶

下载地址 http://archive.apache.org/dist/kafka/2.1.1/

master 快速命令

cd /usr/local/src
wget http://archive.apache.org/dist/kafka/2.1.1/kafka_2.11-2.1.1.tgz
tar xf kafka_2.11-2.1.1.tgz -C /usr/local/
ln -sf /usr/local/kafka_2.11-2.1.1 /usr/local/kafka
cd /usr/local/kafka
mkdir -p logs
cd config
cp server.properties server.properties.ori
# sed -i 's@broker.id=0@broker.id=0@' server.properties
sed -i 's@log.dirs=/tmp/kafka-logs@log.dirs=/usr/local/kafka/logs@g' server.properties
sed -i 's@zookeeper.connect=localhost:2181@zookeeper.connect=master:2181,slave1:2181,slave2:2181@' server.properties
echo " ">>server.properties
echo 'delete.topic.enable=true'>>server.properties 

# 配置kafka环境变量
cat>/etc/profile.d/kafka_env.sh<<EOF
#KAFKA_HOME
export KAFKA_HOME=/usr/local/kafka
export PATH=\$PATH:\$KAFKA_HOME/bin
EOF
source /etc/profile

slave1 快速命令

cd /usr/local/src
wget http://archive.apache.org/dist/kafka/2.1.1/kafka_2.11-2.1.1.tgz
tar xf kafka_2.11-2.1.1.tgz -C /usr/local/
ln -sf /usr/local/kafka_2.11-2.1.1 /usr/local/kafka
cd /usr/local/kafka
mkdir -p logs
cd config
cp server.properties server.properties.ori
sed -i 's@broker.id=0@broker.id=1@' server.properties
sed -i 's@log.dirs=/tmp/kafka-logs@log.dirs=/usr/local/kafka/logs@g' server.properties
sed -i 's@zookeeper.connect=localhost:2181@zookeeper.connect=master:2181,slave1:2181,slave2:2181@' server.properties
echo " ">>server.properties
echo 'delete.topic.enable=true'>>server.properties 

# 配置kafka环境变量
cat>/etc/profile.d/kafka_env.sh<<EOF
#KAFKA_HOME
export KAFKA_HOME=/usr/local/kafka
export PATH=\$PATH:\$KAFKA_HOME/bin
EOF
source /etc/profile

slave2 快速命令

cd /usr/local/src
wget http://archive.apache.org/dist/kafka/2.1.1/kafka_2.11-2.1.1.tgz
tar xf kafka_2.11-2.1.1.tgz -C /usr/local/
ln -sf /usr/local/kafka_2.11-2.1.1 /usr/local/kafka
cd /usr/local/kafka
mkdir -p logs
cd config
cp server.properties server.properties.ori
sed -i 's@broker.id=0@broker.id=2@' server.properties
sed -i 's@log.dirs=/tmp/kafka-logs@log.dirs=/usr/local/kafka/logs@g' server.properties
sed -i 's@zookeeper.connect=localhost:2181@zookeeper.connect=master:2181,slave1:2181,slave2:2181@' server.properties
echo " ">>server.properties
echo 'delete.topic.enable=true'>>server.properties 

# 配置kafka环境变量
cat>/etc/profile.d/kafka_env.sh<<EOF
#KAFKA_HOME
export KAFKA_HOME=/usr/local/kafka
export PATH=\$PATH:\$KAFKA_HOME/bin
EOF 
source /etc/profile

三个机器操作就一个地方不一样就是broker.id不得重复，保证三个机器不一样即可，是整数

master详细操作

[root@master ~]# cd /usr/local/src
[root@master src]# ls
apache-flume-1.6.0-bin         apache-storm-0.9.3.master.tgz  hbase-1.3.1-bin.tar.gz       mysql-connector-java-5.1.46.tar.gz  spark-2.1.1-bin-hadoop2.6.tgz         zkaban
apache-flume-1.6.0-bin.tar.gz  hadoop-2.6.5.tar.gz            jdk-8u172-linux-x64.tar.gz   scala-2.12.4.tgz                    sqoop-1.4.7.bin__hadoop-2.6.0.tar.gz  zookeeper-3.4.5.tar.gz
apache-hive-1.2.2-bin.tar.gz   hbase                          mysql-connector-java-5.1.46  spark-1.6.3-bin-hadoop2.6.tgz       tmp
[root@master src]# wget http://archive.apache.org/dist/kafka/2.1.1/kafka_2.11-2.1.1.tgz
--2019-08-22 16:55:35--  http://archive.apache.org/dist/kafka/2.1.1/kafka_2.11-2.1.1.tgz
Resolving archive.apache.org (archive.apache.org)... 163.172.17.199
Connecting to archive.apache.org (archive.apache.org)|163.172.17.199|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 62090909 (59M) [application/x-gzip]
Saving to: ‘kafka_2.11-2.1.1.tgz’

100%[===================================================================================================================================================>] 62,090,909   138KB/s   in 7m 56s 

2019-08-22 17:03:37 (127 KB/s) - ‘kafka_2.11-2.1.1.tgz’ saved [62090909/62090909]
[root@master src]# tar xf kafka_2.11-2.1.1.tgz -C /usr/local/
[root@master src]# ln -sf /usr/local/kafka_2.11-2.1.1 /usr/local/kafka
[root@master src]# cd /usr/local/kafka
[root@master kafka]# mkdir -p logs
[root@master kafka]# cd config/
[root@master config]# sed -i 's@log.dirs=/tmp/kafka-logs@log.dirs=/usr/local/kafka/logs@g' server.properties
[root@master config]# sed -i 's@zookeeper.connect=localhost:2181@zookeeper.connect=master:2181,slave1:2181,slave2:2181@' server.properties
[root@master config]# echo 'delete.topic.enable=true'>>server.properties 

[root@master ~]# cat>/etc/profile.d/kafka_env.sh<<EOF
> #KAFKA_HOME
> export KAFKA_HOME=/usr/local/kafka
> export PATH=\$PATH:\$KAFKA_HOME/bin
> EOF
[root@master ~]# source /etc/profile

2.2 启动¶

检查zookeeper

root@master:/usr/local/spark# /usr/local/zookeeper/bin/zkServer.sh status
JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Mode: follower

root@slave1:/usr/local/kafka/config# /usr/local/zookeeper/bin/zkServer.sh status
JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Mode: leader

root@slave2:/usr/local/kafka/config# /usr/local/zookeeper/bin/zkServer.sh status
JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Mode: follower

kafka依赖zk，所以在启动之前需要检查 zk是否正常

快速命令

kafka-server-start.sh /usr/local/kafka/config/server.properties

三台机器都要启动此时是前台运行，后面优化这个，这个利于终端打印

启动

[root@master config]# kafka-server-start.sh /usr/local/kafka/config/server.properties
[root@slave1 config]# kafka-server-start.sh /usr/local/kafka/config/server.properties
[root@slave2 config]# kafka-server-start.sh /usr/local/kafka/config/server.properties

检查
[root@master ~]# jps|grep -i kafka
48198 Kafka
[root@slave1 ~]# jps|grep -i kafka
11481 Kafka
[root@slave2 ~]# jps|grep -i kafka
16008 Kafka

3. 启动与停止¶

启动集群

依次在master,slave1,slave2节点上启动kafka

[root@master ~]# kafka-server-start.sh /usr/local/kafka/config/server.properties &
[root@slave1 ~]# kafka-server-start.sh /usr/local/kafka/config/server.properties &
[root@slave2 ~]# kafka-server-start.sh /usr/local/kafka/config/server.properties &

关闭集群

[root@master ~]# kafka-server-stop.sh stop
[root@slave1 ~]# kafka-server-stop.sh stop
[root@slave2 ~]# kafka-server-stop.sh stop