DW(Columnstore)

mysql/mariadb<oltp> as olap:
      缺乏hash join和parallel
      复杂的SQL查询,只会有一个CPU正在满负荷地运转
      方案选型了之后，只能作更多的优化
        尽可能使用主键查询
        选择索引的字段，必定要符合差别率高的字段
        在group by后面使用order by null，禁用默认的group by排序
        手动拆分SQL，再合并结果集
        my.cnf配置优化，增长临时表空间和key_buffer_size

mariadb columnstore (InfiniDB olap store engine)
    大数据离线分析
    大规模分布式并行（MPP架构）
    高性能
    压缩
    列式
  (对比非开源mysql数据仓库系统Infobright)

columnstore:三层架构
      用户模块 User Module：Execution Manager／DMLProc／DDLProc
      性能模块 Performance Module：n*Performance Module
      数据存储层 Data Storage: 本地磁盘(local disks)/共享存储(SAN)/分布式存储(EBS/HDFS/GlusterFS)

columnstore docker install(基于centos的docker):
      git clone https://github.com/mariadb-corporation/mariadb-columnstore-docker.git
      cd mariadb-columnstore-docker
      docker build -t david:mcs .
      docker run -d -p 4306:3306 -v /home/david/dev/dbt3:/mnt/dbt3 --name mcs david:mcs

      execute mcsadmin or mcsimport:docker exec -it mcs bash
      mcsadmin getsystemstatus
      ...

      docker stop mcs
      docker start mcs
      docker rm -v mcs
      docker rmi david:mcs

columnstore vagrant install(基于centos的vm):
　　　 ＃安装virtualbox
      note:disable SecureBoot when boot linux
      sudo apt-get install linux-headers-`uname -r`
      sudo apt-get install virtualbox
      sudo apt-get install dkms
      sudo dpkg-reconfigure virtualbox-dkms
      sudo dpkg-reconfigure virtualbox
      (sudo modprobe vboxdrv)
      (sudo /etc/init.d/vboxdrv setup)
      (sudo cp vboxdrv.ko /lib/modules/)
      (make sure SecureBoot is disabled:sudo mokutil --sb-state)

      vagrant plugin install vagrant-libvirt
      vagrant plugin install vagrant-vbguest

      ＃下载vagrant配置
      git clone https://github.com/mariadb-corporation/mariadb-columnstore-vagrant.git
      cd mariadb-columnstore-vagrant
      vagrant status
      download mariadb-cs-centos-7 and unzip to columnstore

      #启动虚拟机
      (box urls:http://www.vagrantbox.es/)
      vagrant box list

 vagrant box add centos/7 https://github.com/holms/vagrant-centos7-box/releases/download/7.1.1503.001/CentOS-7.1.1503-x86_64-netboot.box --provider=virtualbox
      vagrant up --provider=virtualbox
      (vagrant halt)
      (vagrant suspend)
      (vagrant resume)
      (vagrant reload)
      (vagrant ssh)
      (vagrant destroy)

      #链接mcs
      vagrant ssh
      sudo su -
      mcsadmin
 

columnstore:os准备和dw安装（ubuntu14不支持,未验证）
      vi /etc/hosts
      公私钥认证，打通SSH无密码
      关闭IPTABLES防火墙/关闭SELINUX:vi /etc/selinux/config
      关闭文件系统访问时间和更改磁盘IO算法:vi /etc/fstab; echo "deadline" >   /sys/block/sdb/queue/scheduler
      优化Linux系统内核: vi /etc/sysctl.conf
      调整文件描述符ulimit为65535:vi /etc/security/limits.conf;
      关闭NUMA:vi /etc/grub.conf
      安装jemalloc内存管理器(相似malloc,tcmalloc):jemalloc*
      重启服务器:reboot

      安装boost软件包:
        wget   http://sourceforge.net/projects/boost/files/boost/1.55.0/boost_1_55_0.tar.gz
        tar zxvf boost_1_55_0.tar.gz
        cd boost_1_55_0
        ./bootstrap.sh   --with-libraries=atomic,date_time,exception,filesystem,iostreams,locale,program_options,regex,signals,system,test,thread,timer,log   --prefix=/usr
        ./b2 install

      安装Perl依赖包:
        expect perl perl-DBI openssl zlib perl-DBD-MySQL

      安装mariadb:
        cd /root/
        wget　https://downloads.mariadb.com/enterprise/dapw-ktc5/mariadb-columnstore/1.0.6/centos/x86_64/6/mariadb-columnstore-1.0.6-1-centos6.x86_64.bin.tar.gz
        tar zxvf   mariadb-columnstore-1.0.6-1-centos6.x86_64.bin.tar.gz -C /usr/local/
        /usr/local/mariadb/columnstore/bin/postConfigure

      Report Tool:
       /usr/local/mariadb/columnstore/bin/columnstoreSupport -a
       /usr/local/mariadb/columnstore/bin/columnstoreSupport -a -p 'password'

      Troubleshooting工具：
      　　/usr/local/mariadb/columnstore/bin/editem
          /usr/local/mariadb/columnstore/bin/clearshm
          /usr/local/mariadb/columnstore/bin/configxml.sh
          /usr/local/mariadb/columnstore/bin/dbrmctl
          /usr/local/mariadb/columnstore/bin/viewtablelock
          /usr/local/mariadb/columnstore/bin/cleartablelock

      管理mariadb：
        mcsadmin getSystemStatus
        (cat /root/.bashrc
            alias mcsmysql='/usr/local/mariadb/columnstore/mysql/bin/mysql   --defaults-file=/usr/local/mariadb/columnstore/mysql/my.cnf -u root'
            alias ma=/usr/local/mariadb/columnstore/bin/mcsadmin
            alias mcsadmin=/usr/local/mariadb/columnstore/bin/mcsadmin
            alias home='cd /usr/local/mariadb/columnstore'
            alias log='cd /var/log/mariadb/columnstore/'
            alias core='cd /var/log/mariadb/columnstore/corefiles'
            alias tmsg='tail -f /var/log/messages'
            alias tdebug='tail -f   /var/log/mariadb/columnstore/debug.log'
            alias tinfo='tail -f   /var/log/mariadb/columnstore/info.log'
            alias dbrm='cd   /usr/local/mariadb/columnstore/data1/systemFiles/dbrm'
            alias module='cat   /usr/local/mariadb/columnstore/local/module'
        or
        /usr/local/mariadb/columnstore/bin/columnstoreAlias
        )

        mcsadmin assignPmDBRootConfig 3 pm

        mcsadmin addModule pm2 host1 mypwd
        mcsadmin alterSystem-enableModule pm2
        mcsadmin adddbroot 1
        mcsadmin assigndbrootpmconfig 2 pm2
        mcsadmin startSystem

        mcsadmin addModule um2 host1
        mcsadmin alterSystem-enableModule um2

        mcsadmin removeModule pm2
        mcsadmin removeModule um2

        mcsadmin stopSystem
        mcsadmin startSystem
        mcsadmin restartSystem
        mcsadmin shutdownSystem

        mcsadmin alterSystem-disablemodule pm2,pm3
        mcsadmin alterSystem-enablemodule pm2,pm3
        switchParentOAMModule pm3

        mcsadmin getActiveSQLStatements
        mcsadmin getModuleConfig
        mcsadmin getSoftwareInfo
        mcsadmin getStorageConfig
        mcsadmin getSystemInfo
        mcsadmin getSystemNetworkConfig
        mcsadmin getSystemResourceUsage
        mcsadmin getProcessStatus
        setSystemConfig ExternalMajorThreshold 80

    操做数据库：
      mcsmysql -p'password' db_name

      数据导入mariadb：
        从其余mysql导出：
          select * from sbtest into outfile '/tmp/sbtest.txt'   FIELDS TERMINATED BY ','
          OPTIONALLY ENCLOSED BY '"'  LINES TERMINATED BY '\n';
          #FIELDS TERMINATED BY ',' --字段的结束符
          #OPTIONALLY ENCLOSED BY '"' --字符串的分割符
          #LINES TERMINATED BY '\n' --行的结束符
        导入mcs:
          load data infile '/tmp/sbtest.txt' into table sbtest   FIELDS TERMINATED BY ','  OPTIONALLY   ENCLOSED BY '"'  LINES TERMINATED   BY '\n';

          /usr/local/mariadb/columnstore/bin/cpimport  test    sbtest    /root/tmp/sbtest.txt  -E   '"'  -s ','
          #test是数据库
          #sbtest是表
          #-E是字符串的分割符
          #-s是字段的结束符

      mariadb数据分析：
          mcsmysql test

      mariadb参数调优：
        /usr/local/mariadb/columnstore/etc/Columnstore.xml
        (columnstore不支持主键、索引、timestamp、collate用法、char\varchar类型的sum/average用法)

      mariadb基准测试：
        InfiniDB比InnoDB建表速度慢，语法定制。
        InfiniDB比InnoDB增长列快100倍
        InfiniDB比InnoDB百列字段千万级数据插入快4倍
        InfiniDB比InnoDB百列字段千万级空间少20%
        InfiniDB比InnoDB百列字段千万级查询快100倍
        InfiniDB比InnoDB亿级插入快3倍
        InfiniDB比InnoDB亿级数据空间少50%
        InfiniDB比InnoDB亿级数据查询快150倍
        基于成熟MySQL协议，Tableau能够使用

      mariadb日志：
        /var/log/mariadb/columnstore／*.log

      mariadb扩从节点：
        未知

      其余暂时未知