数据分库拆分测试---dble

时间 2019-12-09

标签数据分库拆分测试 dble 繁體版

原文原文链接

数据库表

server.xml配置

第一段 "< system >" 为DBLE的服务端口（默认8066）和管理端口（默认9066）的配置html

管理端口只能接受DBLE的管理命令，这里不作展开
服务端口即DBLE的业务访问端口，能够接受SQL语句

第二段“< user >”配置管理理用户，默认为man1，密码为654321node

便可以经过 mysql -P9066 -h 127.0.0.1 -u man1 -p654321来下发管理命令

第三段“< user >”配置业务用户，配置了一个帐号test 密码123456,针对数据库testdb,读写权限都有，没有针对表作任何特殊的权限，故把表配置作了注释mysql

<?xml version="1.0" encoding="UTF-8"?>
<!--
  ~ Copyright (C) 2016-2019 ActionTech.
  ~ License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
  -->

<!-- - - Licensed under the Apache License, Version 2.0 (the "License");
    - you may not use this file except in compliance with the License. - You 
    may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 
    - - Unless required by applicable law or agreed to in writing, software - 
    distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT 
    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the 
    License for the specific language governing permissions and - limitations 
    under the License. -->
<!DOCTYPE dble:server SYSTEM "server.dtd">
<dble:server xmlns:dble="http://dble.cloud/" version="2.19.03.0">
    <system>
        <!-- base config -->
        <property name="bindIp">0.0.0.0</property>
        < property name="serverPort">8066</property> 
        <property name="managerPort">9066</property> 
        <property name="maxCon">1024</property> 
        <property name="processors">4</property>
        <property name="backendProcessors">12</property>
        <property name="processorExecutor">4</property> -->
        <property name="backendProcessorExecutor">12</property>
        <property name="complexExecutor">8</property> -->
        <property name="writeToBackendExecutor">4</property>

        <!--<property name="fakeMySQLVersion">5.6.20</property>-->
        <property name="sequnceHandlerType">2</property>
        <!-- serverBacklog size,default 2048-->
        <property name="serverBacklog">2048</property>
        <!--<property name="serverNodeId">1</property>-->
        <!--<property name="showBinlogStatusTimeout">60000</property>-->

        <!--option-->
        <!--<property name="useCompression">1</property>-->
        <!--<property name="usingAIO">0</property>-->
        <!--<property name="useZKSwitch">true</property>-->
        <!--<property name="useThreadUsageStat">0</property>-->
        <!--<property name="usePerformanceMode">0</property>-->

        <!-- query time cost statistics-->
        <!--<property name="useCostTimeStat">0</property>-->
        <!--<property name="maxCostStatSize">100</property>-->
        <!--<property name="costSamplePercent">1</property>-->

        <!--connection -->
        <!--<property name="charset">utf8</property>-->
        <!--<property name="maxPacketSize">16777216</property>-->
        <!--<property name="txIsolation">3</property>-->

        <!--consistency-->
        <!-- check the consistency of table structure between nodes,default not -->
        <property name="checkTableConsistency">0</property>
        <!-- check periodt, he default period is 60000 milliseconds -->
        <property name="checkTableConsistencyPeriod">60000</property>
        <!-- 1 check the consistency of global table, 0 is not -->
        <property name="useGlobleTableCheck">0</property>
        <property name="glableTableCheckPeriod">86400000</property>

        <!-- heartbeat check period -->
        <property name="dataNodeIdleCheckPeriod">300000</property>
        <property name="dataNodeHeartbeatPeriod">10000</property>

        <!-- processor check conn-->
        <property name="processorCheckPeriod">1000</property><!-- unit millisecond -->
        <property name="sqlExecuteTimeout">300</property><!-- unit second -->
        <property name="idleTimeout">1800000</property><!-- unit millisecond -->

        <!-- transaction log -->
        <!-- 1 enable record the transaction log, 0 disable -->
        <property name="recordTxn">0</property>
        <!--<property name="transactionLogBaseDir">/txlogs</property>-->
        <!--<property name="transactionLogBaseName">server-tx</property>-->
        <!--<property name="transactionRatateSize">16</property>--><!-- unit M -->

        <!-- XA transaction -->
        <!-- use XA transaction ,if the mysql service crash,the unfinished XA commit/rollback will retry for several times
       it is the check period for ,default is 1000 milliseconds-->
        <property name="xaSessionCheckPeriod">1000</property>
        <!-- use XA transaction ,the finished XA log will removed. the default period is 1000 milliseconds-->
        <property name="xaLogCleanPeriod">1000</property>
        <!-- XA Recovery Log path -->
        <!--<property name="xaRecoveryLogBaseDir">/tmlogs/</property>-->
        <!-- XA Recovery Log name -->
        <!--<property name="xaRecoveryLogBaseName">tmlog</property>-->

        <!-- true is use JoinStrategy, default false-->
        <property name="useJoinStrategy">true</property>
        <property name="nestLoopConnSize">4</property>
        <property name="nestLoopRowsSize">2000</property>

        <!-- query memory used for per session,unit is M-->
        <property name="otherMemSize">4</property>
        <property name="orderMemSize">4</property>
        <property name="joinMemSize">4</property>

        <!-- off Heap unit:bytes-->
        <property name="bufferPoolChunkSize">4096</property>
        <property name="bufferPoolPageNumber">512</property>
        <property name="bufferPoolPageSize">2097152</property>
        <!--<property name="mappedFileSize">2097152</property>-->

        <!-- sql statistics-->
        <!-- 1 means use SQL statistics, 0 means not -->
        <property name="useSqlStat">0</property>
        <!--<property name="bufferUsagePercent">80</property>-->
        <!--<property name="clearBigSqLResultSetMapMs">600000</property>-->
        <!--<property name="sqlRecordCount">10</property>-->
        <!--<property name="maxResultSet">524288</property>-->

        <!-- backSocket unit:bytes-->
        <!--<property name="backSocketSoRcvbuf">4194304</property>-->
        <!--<property name="backSocketSoSndbuf">1048576</property>-->
        <!--<property name="backSocketNoDelay">1</property>-->

        <!-- frontSocket-->
        <!--<property name="frontSocketSoRcvbuf">1048576</property>-->
        <!--<property name="frontSocketSoSndbuf">4194304</property>-->
        <!--<property name="frontSocketNoDelay">1</property>-->

        <!-- for join tmp results -->
        <!--<property name="mergeQueueSize">1024</property>-->
        <!--<property name="orderByQueueSize">1024</property>-->
        <!--<property name="joinQueueSize">1024</property>-->

        <!-- if enable the slow query log -->
        <property name="enableSlowLog">0</property>
        <!-- the slow query log location -->
        <!--<property name="slowLogBaseDir">./slowlogs</property>-->
        <!--<property name="slowLogBaseName">slow-query</property>-->
        <!-- the max period for flushing the slow query log from memory to disk  after last time , unit is second -->
        <property name="flushSlowLogPeriod">1</property>
        <!-- the max records for flushing the slow query log from memory to disk after last time -->
        <property name="flushSlowLogSize">1000</property>
        <!-- the threshold for judging if the query is slow , unit is millisecond -->
        <property name="sqlSlowTime">100</property>


    </system>

    <!-- firewall config -->
    <!--
    <firewall>
    <whitehost>
          <host host="127.0.0.1" user="root"/>
          <host host="0:0:0:0:0:0:0:1" user="root"/>
       </whitehost>
       <blacklist check="true">
       <property name="selelctAllow">false</property>
       </blacklist>
    </firewall>
    -->
    <user name="man1">
        <property name="password">654321</property>
        <property name="manager">true</property>
        <!-- manager user can't set schema-->
    </user>

    <user name="test">
        <property name="password">123456</property>
        <property name="schemas">TESTDB</property>

        <!-- table's DML privileges  INSERT/UPDATE/SELECT/DELETE -->
        <!--
        <privileges check="false">
            <schema name="TESTDB" dml="0110" >
                <table name="tb01" dml="0000"></table>
                <table name="tb02" dml="1111"></table>
            </schema>
        </privileges>
         -->
    </user>


    <!--<alarm>
        <url>10.186.22.103</url>
        <port>5700</port>
        <level>warn</level>
        <serverId>dbleServer</serverId>
        <componentId>componentId</componentId>
        <componentType>componentType</componentType>
    </alarm>-->

</dble:server>

schema.xml

users用户表按照取模的方式平均拆分到了MySQL A和MySQL B两个数据数据库实例上算法

schema 逻辑数据库信息，此数据库为逻辑数据库，name 与 server.xml 中 schema对应;
dataNode 分片信息，此为分片节点的定义；分片名字和schema的dataNode对应；分片与下面的dataHost 物理实例进行关联；
dataHost 物理实例组信息，dataHost下能够挂载同组的读写物理实例节点，实现高可用或者读写分离；

每一个节点的属性逐一说明：sql

schema:

属性说明：数据库

- name 逻辑数据库名，与 server.xml 中的 schema 对应； - table:

子属性说明：express

    -  name 表名，物理数据库中表名
    -  dataNode 表存储到哪些节点，多个节点用逗号分隔
    -  primaryKey 主键，用于主键缓存和自增识别，不做主键约束
    -  autoIncrement 是否自增 
    -  rule 分片规则名，具体规则下文 rule 详细介绍

dataNode
属性说明：apache

name 节点名，与 table 中 dataNode 对应
datahost 物理实例组名，与 datahost 中 name 对应
database 物理数据库中数据库名；

dataHost

属性说明：segmentfault

name 物理数据库名，与 dataNode 中 dataHost 对应
balance 均衡负载的方式
switchtype 写节点的高可用切换方式；等于1时，心跳不健康发生切换
heartbeat 心跳检测语句，注意语句结尾的分号要加
writehost 写物理实例

```
子属性说明 ：
```

-  host 物理实例名
-  url 物理库IP+Port
-  user 物理库用户
-  password 物理库密码

<?xml version="1.0"?>
<!--
  ~ Copyright (C) 2016-2019 ActionTech.
  ~ License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
  -->

<!DOCTYPE dble:schema SYSTEM "schema.dtd">
<dble:schema xmlns:dble="http://dble.cloud/" version="2.19.03.0">

    <schema name="testdb">
        <table name="users" primaryKey="ID"  dataNode="dn1,dn2" rule="sharding-by-mod2" />
    </schema>
    
    <dataNode name="dn1" dataHost="Group1" database="testdb"/>
    <dataNode name="dn2" dataHost="Group2" database="testdb"/>
    
    <!-- 物理数据库配置 -->
    <dataHost name="Group1" maxCon="1000" minCon="10" balance="0" switchType="1" slaveThreshold="100">
        <heartbeat>show slave status</heartbeat>
        <writeHost host="MySQLA" url="192.168.1.53:3306" user="root" password="123456"/>
    </dataHost>

    <dataHost name="Group2" maxCon="1000" minCon="10" balance="0" switchType="1" slaveThreshold="100">
        <heartbeat>show slave status</heartbeat>
         <writeHost host="MySQLA" url="192.168.1.116:3306" user="root" password="123456"/>
    </dataHost>
    
</dble:schema>

rule.xml

主要关注rule属性，rule属性的内容来源于rule.xml这个文件，DBLE支持多种分表分库的规则后端

这里选择的sharding-by-mod2，是hash算法的特例，就是将数据平均拆分。由于我后端是两台物理库，因此rule.xml中hashmod2对应的partitionCountt为2

<?xml version="1.0" encoding="UTF-8"?>
<!--
  ~ Copyright (C) 2016-2019 ActionTech.
  ~ License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
  -->

<!-- - - Licensed under the Apache License, Version 2.0 (the "License");
    - you may not use this file except in compliance with the License. - You 
    may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 
    - - Unless required by applicable law or agreed to in writing, software - 
    distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT 
    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the 
    License for the specific language governing permissions and - limitations 
    under the License. -->
<!DOCTYPE dble:rule SYSTEM "rule.dtd">
<dble:rule xmlns:dble="http://dble.cloud/" version="2.19.03.0">
    <tableRule name="sharding-by-mod2">
        <rule>
            <columns>id</columns>
            <algorithm>hashmod2</algorithm>
        </rule>
    </tableRule>
    <function name="hashmod2" class="Hash">
        <property name="partitionCount">2</property>
        <property name="partitionLength">1</property>
    </function>

</dble:rule>

运行结果

运行bin里都dble

链接数据库

插入2条数据

查看数据库里面结果

参考https://segmentfault.com/a/1190000018019509?utm_source=tag-newest