canal动态监控Mysql，将binlog日志解析后，把采集到的数据发送到Kafka

时间 2021-04-13

标签 java mysql git github sql 数据库 apache json bootstrap api 栏目 MySQL 繁體版

原文原文链接

生产者要将发送的数据转化为字节数组才能经过网络发动给Kafka，对于一些简单的数据，Kafka自带了一些序列化工具。java

//建立生产者实例
private static Producer<String , String> createProducer(){
    Properties properties = new Properties();

    properties.put("metadata.broker.list" , GlobalConfigUtil.kafkaBootstrap);
    properties.put("zookeeper.connect" , GlobalConfigUtil.kafkaZookeeper);
    properties.put("serializer.class" , StringEncoder.class.getName());
    
    return new Producer<String, String>(new ProducerConfig(properties));
}

在一般的微服务中，服务之间须要频繁的传递各类负责的数据结构，可是kafka仅仅支持简单的类型如String，Integer。因而咱们在服务之间使用JSONObject，由于JSON能够很容易的转化为String，而String的序列化和反序列化已经被支持。mysql

JSONObject jsonObject = new JSONObject();
jsonObject.put("logFileName", logFileName);
jsonObject.put("logFileOffset", logFileOffset);
jsonObject.put("dbName", dbName);
jsonObject.put("tableName", tableName);
jsonObject.put("eventType", eventType);
jsonObject.put("columnValueList", columnValueList);
jsonObject.put("emptyCount", emptyCount);
jsonObject.put("timestamp", timestamp);

//拼接全部binlog解析的字段
String data = JSON.toJSONString(jsonObject);

// 解析后的数据发送到kafka
KafkaSender.sendMessage(GlobalConfigUtil.kafkaInput, JSON.toJSONString(key), data);

ResourceBundle类是用来读取propertise资源文件的，能够在初始化时把配置项所有一次读入，并保存在静态成员变量中。避免每次须要的时候才去读取相关配置文件的class，I/O速度慢，容易形成性能上的瓶颈。git

//读取application.properties文件
private static ResourceBundle resourceBundle = ResourceBundle.getBundle("application");

public static String canalHost= resourceBundle.getString("canal.host");
public static String canalPort = resourceBundle.getString("canal.port");
public static String canalInstance = resourceBundle.getString("canal.instance");
public static String mysqlUsername = resourceBundle.getString("mysql.username");
public static String mysqlPassword=  resourceBundle.getString("mysql.password");
public static String kafkaBootstrap= resourceBundle.getString("kafka.bootstrap.servers");
public static String kafkaZookeeper= resourceBundle.getString("kafka.zookeeper.connect");
public static String kafkaInput = resourceBundle.getString("kafka.input.topic");

完整代码github

#pom文件
<dependency>  
    <groupId>com.alibaba.otter</groupId>  
    <artifactId>canal.client</artifactId>  
    <version>1.0.24</version>  
</dependency>  
<!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka -->  
<dependency>  
    <groupId>org.apache.kafka</groupId>  
    <artifactId>kafka_2.11</artifactId>  
    <version>0.9.0.1</version>  
    <exclusions>  
        <exclusion>  
            <groupId>org.slf4j</groupId>  
            <artifactId>slf4j-log4j12</artifactId>  
        </exclusion>  
    </exclusions>  
</dependency>  
  
<!--对象和json 互相转换的-->  
<dependency>  
    <groupId>com.alibaba</groupId>  
    <artifactId>fastjson</artifactId>  
    <version>1.2.44</version>  
</dependency>

import java.util.Locale;  
import java.util.ResourceBundle;  
  
/**  
 * 配置文件的公共类  
 */  
public class GlobalConfigUtil {  
  
    //读取application.properties文件  
 private static ResourceBundle resourceBundle = ResourceBundle.getBundle("application");  
  
    public static String canalHost= resourceBundle.getString("canal.host");  
    public static String canalPort = resourceBundle.getString("canal.port");  
    public static String canalInstance = resourceBundle.getString("canal.instance");  
    public static String mysqlUsername = resourceBundle.getString("mysql.username");  
    public static String mysqlPassword=  resourceBundle.getString("mysql.password");  
    public static String kafkaBootstrap= resourceBundle.getString("kafka.bootstrap.servers");  
    public static String kafkaZookeeper= resourceBundle.getString("kafka.zookeeper.connect");  
    public static String kafkaInput = resourceBundle.getString("kafka.input.topic");  
  
    public static void main(String[] args) {  
        System.out.println(canalHost);  
    }  
}

import kafka.javaapi.producer.Producer;  
import kafka.producer.KeyedMessage;  
import kafka.producer.ProducerConfig;  
import kafka.serializer.StringEncoder;  
  
import java.util.Properties;  
  
/**  
 * Kafka生产消息工具类  
 */  
public class KafkaSender {  
    private String topic;  
  
    public KafkaSender(String topic){  
        super();  
        this.topic = topic;  
    }  
  
    /**  
 * 发送消息到Kafka指定topic  
 * * @param topic topic名字  
 * @param key 键值  
 * @param data 数据  
 */  
 public static void sendMessage(String topic , String key , String data){  
        Producer<String, String> producer = createProducer();  
        producer.send(new KeyedMessage<String , String>(topic , key , data));  
    }  
  
    /**  
 * 建立生产者实例  
 * @return  
 */  
 private static Producer<String , String> createProducer(){  
        Properties properties = new Properties();  
  
        properties.put("metadata.broker.list" , GlobalConfigUtil.kafkaBootstrap);  
        properties.put("zookeeper.connect" , GlobalConfigUtil.kafkaZookeeper);  
        properties.put("serializer.class" , StringEncoder.class.getName());  
  
        return new Producer<String, String>(new ProducerConfig(properties));  
    }  
}

import com.alibaba.fastjson.JSON;  
import com.alibaba.fastjson.JSONObject;  
import com.alibaba.otter.canal.client.CanalConnector;  
import com.alibaba.otter.canal.client.CanalConnectors;  
import com.alibaba.otter.canal.protocol.CanalEntry;  
import com.alibaba.otter.canal.protocol.Message;  
  
import java.net.InetSocketAddress;  
import java.util.ArrayList;  
import java.util.List;  
import java.util.UUID;  
  
/**  
 * Canal解析binlog日志工具类  
 */  
public class CanalClient {  
  
    static class ColumnValuePair {  
        private String columnName;  
        private String columnValue;  
        private Boolean isValid;  
  
        public ColumnValuePair(String columnName, String columnValue, Boolean isValid) {  
            this.columnName = columnName;  
            this.columnValue = columnValue;  
            this.isValid = isValid;  
        }  
  
        public String getColumnName() { return columnName; }  
        public void setColumnName(String columnName) { this.columnName = columnName; }  
        public String getColumnValue() { return columnValue; }  
        public void setColumnValue(String columnValue) { this.columnValue = columnValue; }  
        public Boolean getIsValid() { return isValid; }  
        public void setIsValid(Boolean isValid) { this.isValid = isValid; }  
    }  
  
    /**  
 * 获取Canal链接  
 *  
 * @param host 主机名  
 * @param port 端口号  
 * @param instance Canal实例名  
 * @param username 用户名  
 * @param password 密码  
 * @return Canal链接器  
 */  
 public static CanalConnector getConn(String host, int port, String instance, String username, String password) {  
        CanalConnector canalConnector = CanalConnectors.newSingleConnector(new InetSocketAddress(host, port), instance, username, password);  
        return canalConnector;  
    }  
  
    /**  
 * 解析Binlog日志  
 *  
 * @param entries Binlog消息实体  
 * @param emptyCount 操做的序号  
 */  
 public static void analysis(List<CanalEntry.Entry> entries, int emptyCount) {  
        for (CanalEntry.Entry entry : entries) {  
            // 只解析mysql事务的操做，其余的不解析  
 if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN ||  
                    entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {  
                continue;  
            }  
  
            // 那么解析binlog  
 CanalEntry.RowChange rowChange = null;  
  
            try {  
                rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());  
            } catch (Exception e) {  
                e.printStackTrace();  
            }  
  
            // 获取操做类型字段（增长  删除  修改）  
 CanalEntry.EventType eventType = rowChange.getEventType();  
            // 获取binlog文件名称  
 String logfileName = entry.getHeader().getLogfileName();  
            // 读取当前操做在binlog文件的位置  
 long logfileOffset = entry.getHeader().getLogfileOffset();  
            // 获取当前操做所属的数据库  
 String dbName = entry.getHeader().getSchemaName();  
            // 获取当前操做所属的表  
 String tableName = entry.getHeader().getTableName();//当前操做的是哪一张表  
 long timestamp = entry.getHeader().getExecuteTime();//执行时间  
  
 // 解析操做的行数据  
 for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {  
                // 删除操做  
 if (eventType == CanalEntry.EventType.DELETE) {  
                    // 获取删除以前的全部列数据  
 dataDetails(rowData.getBeforeColumnsList(), logfileName, logfileOffset, dbName, tableName, eventType, emptyCount,timestamp);  
                }  
                // 新增操做  
 else if (eventType == CanalEntry.EventType.INSERT) {  
                    // 获取新增以后的全部列数据  
 dataDetails(rowData.getAfterColumnsList(), logfileName, logfileOffset, dbName, tableName, eventType, emptyCount,timestamp);  
                }  
                // 更新操做  
 else {  
                    // 获取更新以后的全部列数据  
 dataDetails(rowData.getAfterColumnsList(), logfileName, logfileOffset, dbName, tableName, eventType, emptyCount,timestamp);  
                }  
            }  
        }  
    }  
  
    /**  
 * 解析具体一条Binlog消息的数据  
 *  
 * @param columns 当前行全部的列数据  
 * @param logFileName binlog文件名  
 * @param logFileOffset 当前操做在binlog中的位置  
 * @param dbName 当前操做所属数据库名称  
 * @param tableName 当前操做所属表名称  
 * @param eventType 当前操做类型（新增、修改、删除）  
 * @param emptyCount 操做的序号  
 */  
 private static void dataDetails(List<CanalEntry.Column> columns,  
                                    String logFileName,  
                                    Long logFileOffset,  
                                    String dbName,  
                                    String tableName,  
                                    CanalEntry.EventType eventType,  
                                    int emptyCount,  
                                    long timestamp) {  
  
        // 找到当前那些列发生了改变  以及改变的值  
 List<ColumnValuePair> columnValueList = new ArrayList<ColumnValuePair>();  
  
        for (CanalEntry.Column column : columns) {  
            ColumnValuePair columnValuePair = new ColumnValuePair(column.getName(), column.getValue(), column.getUpdated());  
            columnValueList.add(columnValuePair);  
        }  
  
        String key = UUID.randomUUID().toString();  
        JSONObject jsonObject = new JSONObject();  
//        jsonObject.put("logFileName", logFileName);  
//        jsonObject.put("logFileOffset", logFileOffset);  
 jsonObject.put("dbName", dbName);  
        jsonObject.put("tableName", tableName);  
        jsonObject.put("eventType", eventType);  
        jsonObject.put("columnValueList", columnValueList);  
//        jsonObject.put("emptyCount", emptyCount);  
//        jsonObject.put("timestamp", timestamp);  
  
  
 // 拼接全部binlog解析的字段  
 String data = JSON.toJSONString(jsonObject);  
  
        System.out.println("【JSON】" + data);  
  
        // 解析后的数据发送到kafka  
 KafkaSender.sendMessage(GlobalConfigUtil.kafkaInput, JSON.toJSONString(key), data);  
    }  
  
    /**  
 * 客户端入口方法  
 * @param args  
 */  
 public static void main(String[] args) {  
        // 加载配置文件  
 String host = GlobalConfigUtil.canalHost;  
        int port = Integer.parseInt(GlobalConfigUtil.canalPort);  
        String instance = GlobalConfigUtil.canalInstance;  
        String username = GlobalConfigUtil.mysqlUsername;  
        String password = GlobalConfigUtil.mysqlPassword;  
  
        // 获取Canal链接  
 CanalConnector conn = getConn(host, port, instance, username, password);  
  
        // 从binlog中读取数据  
 int batchSize = 100;  
        int emptyCount = 1;  
  
        try {  
            conn.connect();  
            conn.subscribe(".*..*");  
            conn.rollback();  
  
            int totalCount = 120; //循环次数  
  
 while (emptyCount < totalCount) {  
                // 获取数据  
 Message message = conn.getWithoutAck(batchSize);  
  
                long id = message.getId();  
                int size = message.getEntries().size();  
                if (id == -1 || size == 0) {  
                    emptyCount=0;  
                    //没有读取到任何数据  
 System.out.println("目前没有读取到任何数据...");  
                } else {  
                    //有数据，那么解析binlog日志  
 analysis(message.getEntries(), emptyCount);  
                    emptyCount++;  
                }  
                // 确认消息  
 conn.ack(message.getId());  
            }  
        } catch (Exception e) {  
            e.printStackTrace();  
        } finally {  
            conn.disconnect();  
        }  
    }  
}

#application.properties, 如下请更改成自已的数据库信息  
canal.host=xxx.xx.xxx.xxx  
canal.port=11111  
canal.instance=example  
mysql.username=root  
mysql.password=xxxxxx  
kafka.bootstrap.servers = xxx.xx.xxx.xxx:9092  
kafka.zookeeper.connect = xxx.xx.xxx.xxx:2182  
kafka.input.topic=test

具体代码请移步：SimpleMysqlCanalKafkaSamplesql