目录
sqoop%20%E5%AE%89%E8%A3%85-toc" style="margin-left:40px;">sqoop 安装
配置 mysql
sqoop%20%E5%AE%89%E8%A3%85-toc" style="margin-left:80px;">sqoop 安装
sqoop%20%E6%8C%87%E4%BB%A4%E9%9B%86-toc" style="margin-left:40px;">sqoop 指令集
sqoop%20%E4%BD%BF%E7%94%A8-toc" style="margin-left:40px;">sqoop 使用
sqoop%20%E5%88%9B%E5%BB%BA%20hive%20%E8%A1%A8-toc" style="margin-left:80px;">sqoop 创建 hive 表
sqoop%20%E5%85%A8%E9%87%8F%E5%AF%BC%E5%85%A5%E8%A1%A8-toc" style="margin-left:80px;">sqoop 全量导入表
sqoop%20%E5%A2%9E%E9%87%8F%E5%AF%BC%E5%85%A5%E8%A1%A8-toc" style="margin-left:80px;">sqoop 增量导入表
sqoop%20%E5%85%A8%E9%87%8F%E5%AF%BC%E5%87%BA%E8%A1%A8-toc" style="margin-left:80px;">sqoop 全量导出表
sqoop%20%E5%88%86%E5%8C%BA%E8%A1%A8%E5%AF%BC%E5%85%A5%E8%A1%A8-toc" style="margin-left:80px;">sqoop 分区表导入表
sqoop%20%E5%88%86%E5%8C%BA%E8%A1%A8%E5%AF%BC%E5%87%BA%E8%A1%A8-toc" style="margin-left:80px;">sqoop 分区表导出表
上机练习
sqoop%20%E5%AE%89%E8%A3%85">sqoop 安装
配置 mysql
sqoop 安装
sqoop%20%E6%8C%87%E4%BB%A4%E9%9B%86">sqoop 指令集
sqoop%20%E4%BD%BF%E7%94%A8">sqoop 使用
sqoop%20%E5%88%9B%E5%BB%BA%20hive%20%E8%A1%A8">sqoop 创建 hive 表
sqoop%20%E5%85%A8%E9%87%8F%E5%AF%BC%E5%85%A5%E8%A1%A8">sqoop 全量导入表
sqoop%20%E5%A2%9E%E9%87%8F%E5%AF%BC%E5%85%A5%E8%A1%A8">sqoop 增量导入表
sqoop%20%E5%85%A8%E9%87%8F%E5%AF%BC%E5%87%BA%E8%A1%A8">sqoop 全量导出表
--export-dir '/user/hive/warehouse/db_hive.db/student2'
sqoop%20%E5%88%86%E5%8C%BA%E8%A1%A8%E5%AF%BC%E5%85%A5%E8%A1%A8">sqoop 分区表导入表
sqoop%20%E5%88%86%E5%8C%BA%E8%A1%A8%E5%AF%BC%E5%87%BA%E8%A1%A8">sqoop 分区表导出表
上机练习
#!/bin/bash
sqoop export \
--connect
"jdbc:mysql://hadoop100:3306/test?useUnicode=true&characterEnc
oding=utf-8" \
--username test \
--password test \--table student2 \
--export-dir /user/hive/warehouse/db_hive.db/student2 \
--fields-terminated-by "," \
-m 1
#!/bin/bash
sqoop import \
--connect
"jdbc:mysql://hadoop100:3306/test?useUnicode=true&characterEnc
oding=utf-8" \
--username test \
--password test \
--table student2 \
--delete-target-dir \
--create-hive-table \
--hive-import \
--hive-database db_hive \
--hive-overwrite \
--hive-table student2_sqoop \
--target-dir /user/hive/warehouse/db_hive.db/student2_sqoop \
--input-fields-terminated-by ',' \
-m 1
#!/bin/bash
sqoop_comm(){
sqoop export \
--connect
"jdbc:mysql://hadoop100:3306/test?useUnicode=true&characterEnc
oding=utf-8" \
--username test \
--password test \
--table log_ \
--export-dir /user/hive/warehouse/db_hive.db/log/$part \
--fields-terminated-by '\t' \
-m 1
}
for part in \
`beeline -u "jdbc:hive2://hadoop100:10000/db_hive" \
--showHeader=False --outputformat=csv2 \
-n root -p 123456 -e 'show partitions log'`
do
echo "-----------$part-----------"
sqoop_comm
done
#!/bin/bash
sqoop_comm(){
sqoop import \
--connect
"jdbc:mysql://hadoop100:3306/test?useUnicode=true&characterEnc
oding=utf-8" \
--username test \
--password test \
--query "select * from log_ where date_format(cast(date_ as
date),'%Y%m%d')=$part and \$CONDITIONS" \
--delete-target-dir \
--create-hive-table \
--hive-import \
--hive-overwrite \
--hive-table sqoop_log \
--hive-database db_hive \
--hive-partition-key load_date \
--hive-partition-value $part \
--target-dir
/user/hive/warehouse/db_hive.db/sqoop_log/load_date=$part \
--input-fields-terminated-by '\t' \
-m 1
}
for part in \
`mysql -utest -ptest --database=test -Ne \"select distinct DATE_FORMAT(cast(date_ as date),'%Y%m%d')from
log_"`
do
echo "---------$part---------"
sqoop_comm
done
beeline -u "jdbc:hive2://hadoop100:10000/db_hive" \
--showHeader=False --outputformat=csv2 \
-n root -p 123456 -e 'msck repair table sqoop_log