Flume 将数据导入Hive数据库
1:建立.conf文件apache
#cd /usr/tmpvim
#mkdir flumesrcoop
#vim netcat_to_hive_user.confspa
a1.sources=r13d
a1.sinks=s1
a1.channels=c1blog
a1.sources.r1.type=netcat
a1.sources.r1.bind=master
a1.sources.r1.port=44444hadoop
a1.sinks.s1.type=hive
a1.sinks.s1.channel = c1
a1.sinks.s1.hive.metastore = thrift://master:9083
a1.sinks.s1.hive.database = bd18
a1.sinks.s1.hive.table = flume_user
a1.sinks.s1.useLocalTimeStamp = false
a1.sinks.s1.serializer = DELIMITED
a1.sinks.s1.serializer.delimiter = "\t"
a1.sinks.s1.serializer.serdeSeparator = '\t'
a1.sinks.s1.serializer.fieldnames=user_id,user_name,age
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
ci
2:运行.conf文件string
#flume-ng agent -c conf -f netcat_to_hive_user.conf --name a1 -Dflume.root.logger=INFO,console
3:将hive的四个jar包复制到flume安装目录的lib文件夹下
4:启动hive的metastore
#hive --service metastore
5:进入hive,建立数据库和表
hive> create database bd18;
hive> create table flume_user(
> user_id int,
> user_name string,
> age int
> )
> clustered by(user_id) into 2 buckets
> stored as orc
> tblproperties("transactional"='true');
hive> select * from flume_user;
FAILED: SemanticException [Error 10265]: This command is not allowed on an ACID table bd18.flume_user with a non-ACID transaction manager. Failed command: select * from flume_user
hive> set hive.support.concurrency=true;
hive> set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
hive> select * from flume_user;
OK
Time taken: 5.809 seconds
6:操做flume,向hive输入数据
[root@master ~]# telnet master 44444
Trying 192.168.2.100...
Connected to master.
Escape character is '^]'.
1 jim 18
OK
注:中间用Tab键隔开
7:查询Hive表中的内容