datart读取hive数据时,需要先在datart的lib目录下导入hive jdbc相关的包,这里面有几个坑记录下:
1.和springboot中commons-lang3冲突
2.hive中带的jetty和springboot冲突
3.hive jdbc的包的版本号一定要小于登录hive服务端的版本,否则会报Required field ‘client_protocol’ is unset的错误,在引入hive jdbc包的时候,要先查看hive的版本,具体方法:可以找到hive的安装目录 whereis hive,查看hive lib目录下相关包的版本或者执行hive后,通过查看日志得到hive的版本。
最后在导入包的时候,一个个导比较麻烦,而且针对jar包冲突请求也不好处理,所以新建了个项目将所需的包引入,使用maven-shade-plugin插件进行打包并对冲突了的jar包进行重命名,消除冲突,最后的pom文件如下:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"><modelVersion>4.0.0</modelVersion><groupId>org.example</groupId><artifactId>hive-dependency</artifactId><version>1.0-SNAPSHOT</version><properties><maven.compiler.source>8</maven.compiler.source><maven.compiler.target>8</maven.compiler.target><hive.version>1.1.0</hive.version></properties><dependencies><!-- https://mvnrepository.com/artifact/org.apache.hive/hive-jdbc --><dependency><groupId>org.apache.hive</groupId><artifactId>hive-jdbc</artifactId><version>${hive.version}</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-common</artifactId><version>2.6.0</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-client</artifactId><version>2.7.1</version></dependency><dependency><groupId>org.apache.hive</groupId><artifactId>hive-exec</artifactId><version>${hive.version}</version><exclusions><exclusion><artifactId>commons-lang3</artifactId><groupId>org.apache.commons</groupId></exclusion></exclusions></dependency></dependencies><build><plugins><plugin><artifactId>maven-compiler-plugin</artifactId><configuration><source>1.8</source><target>1.8</target><encoding>UTF-8</encoding></configuration></plugin><plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-shade-plugin</artifactId><version>2.4.3</version><executions><execution><phase>package</phase><goals><goal>shade</goal></goals><configuration><filters><filter><artifact>*:*</artifact><excludes><exclude>META-INF/*.SF</exclude><exclude>META-INF/*.DSA</exclude><exclude>META-INF/*.RSA</exclude><exclude>org/eclipse/*</exclude></excludes></filter></filters><relocations><relocation><pattern>org.apache.commons.lang3</pattern><shadedPattern>org.apache.commons.hive.lang3</shadedPattern></relocation><relocation><pattern>org.eclipse.jetty</pattern><shadedPattern>org.eclipse.hive.jetty</shadedPattern></relocation><relocation><pattern>org.mortbay</pattern><shadedPattern>org.mortbay.hive</shadedPattern></relocation></relocations></configuration></execution></executions></plugin></plugins></build><repositories><repository><id>aliyun</id><!-- <mirrorOf>*</mirrorOf>--><name>spring-plugin</name><url>https://maven.aliyun.com/repository/spring-plugin</url><releases><enabled>true</enabled></releases><snapshots><enabled>false</enabled></snapshots></repository></repositories>
</project>