问题描述

在IEDA连接虚拟机上的Hive报的
Exception in thread “main” org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient
at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:110)
at org.apache.spark.sql.hive.HiveExternalCatalog.databaseExists(HiveExternalCatalog.scala:223)
at org.apache.spark.sql.internal.SharedState.externalCatalog $l zyco m p u t e (S ha re d St a t e . sc a l a : 150) a t or g . a p a c h e . s p a r k . s ql . in t er na l . S ha re d St a t e . e x t er na lC a t a l o g (S ha re d St a t e . sc a l a : 140) a t or g . a p a c h e . s p a r k . s ql . in t er na l . S ha re d St a t e . g l o ba lT e m p Vi e wM ana g er$ lzycompute(SharedState.scala:170)
at org.apache.spark.sql.internal.SharedState.globalTempViewManager(SharedState.scala:168)
at org.apache.spark.sql.hive.HiveSessionStateBuilder. $an o n f u n$ catalog $2 (H i v e S ess i o n St a t e B u i l d er . sc a l a : 70) a t or g . a p a c h e . s p a r k . s ql . c a t a l ys t . c a t a l o g . S ess i o n C a t a l o g . g l o ba lT e m p Vi e wM ana g er$ lzycompute(SessionCatalog.scala:122)
at org.apache.spark.sql.catalyst.catalog.SessionCatalog.globalTempViewManager(SessionCatalog.scala:122)
at org.apache.spark.sql.catalyst.catalog.SessionCatalog.listTables(SessionCatalog.scala:1031)
at org.apache.spark.sql.catalyst.catalog.SessionCatalog.listTables(SessionCatalog.scala:1017)
at org.apache.spark.sql.catalyst.catalog.SessionCatalog.listTables(SessionCatalog.scala:1009)
at org.apache.spark.sql.execution.datasources.v2.V2SessionCatalog.listTables(V2SessionCatalog.scala:57)
at org.apache.spark.sql.execution.datasources.v2.ShowTablesExec.run(ShowTablesExec.scala:40)
at org.apache.spark.sql.execution.datasources.v2.V2CommandExec.result $l zyco m p u t e (V 2 C o mman d E x ec . sc a l a : 43) a t or g . a p a c h e . s p a r k . s ql . e x ec u t i o n . d a t a so u rces . v 2. V 2 C o mman d E x ec . res u lt (V 2 C o mman d E x ec . sc a l a : 43) a t or g . a p a c h e . s p a r k . s ql . e x ec u t i o n . d a t a so u rces . v 2. V 2 C o mman d E x ec . e x ec u t e C o ll ec t (V 2 C o mman d E x ec . sc a l a : 49) a t or g . a p a c h e . s p a r k . s ql . e x ec u t i o n . Q u ery E x ec u t i o n$ $an o n f u n$ eagerlyExecuteCommands $1.$ anonfun$applyOrElse $1 (Q u ery E x ec u t i o n . sc a l a : 98) a t or g . a p a c h e . s p a r k . s ql . e x ec u t i o n . SQ L E x ec u t i o n$ . $an o n f u n$ withNewExecutionId $6 (SQ L E x ec u t i o n . sc a l a : 109) a t or g . a p a c h e . s p a r k . s ql . e x ec u t i o n . SQ L E x ec u t i o n$ .withSQLConfPropagated(SQLExecution.scala:169)
at org.apache.spark.sql.execution.SQLExecution $.$ anonfun$withNewExecutionId $1 (SQ L E x ec u t i o n . sc a l a : 95) a t or g . a p a c h e . s p a r k . s ql . Sp a r k S ess i o n . w i t h A c t i v e (Sp a r k S ess i o n . sc a l a : 779) a t or g . a p a c h e . s p a r k . s ql . e x ec u t i o n . SQ L E x ec u t i o n$ .withNewExecutionId(SQLExecution.scala:64)
at org.apache.spark.sql.execution.QueryExecution $KaTeX parse error: Can't use function '$' in math mode at position 8: anonfun$̲eagerlyExecuteC\dots$ anonfun$eagerlyExecuteCommands $1. a ppl y O r El se (Q u ery E x ec u t i o n . sc a l a : 94) a t or g . a p a c h e . s p a r k . s ql . c a t a l ys t . t rees . T ree N o d e .$ anonfun$transformDownWithPruning $1 (T ree N o d e . sc a l a : 584) a t or g . a p a c h e . s p a r k . s ql . c a t a l ys t . t rees . C u rre n tO r i g in$ .withOrigin(TreeNode.scala:176)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:584)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org $a p a c h e$ spark $s ql$ catalyst $pl an s$ logical $A na l ys i sHe lp er$ $s u p er$ transformDownWithPruning(LogicalPlan.scala:30)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:267)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning $(A na l ys i sHe lp er . sc a l a : 263) a t or g . a p a c h e . s p a r k . s ql . c a t a l ys t . pl an s . l o g i c a l . L o g i c a lPl an . t r an s f or m Do w nWi t h P r u nin g (L o g i c a lPl an . sc a l a : 30) a t or g . a p a c h e . s p a r k . s ql . c a t a l ys t . pl an s . l o g i c a l . L o g i c a lPl an . t r an s f or m Do w nWi t h P r u nin g (L o g i c a lPl an . sc a l a : 30) a t or g . a p a c h e . s p a r k . s ql . c a t a l ys t . t rees . T ree N o d e . t r an s f or m Do w n (T ree N o d e . sc a l a : 560) a t or g . a p a c h e . s p a r k . s ql . e x ec u t i o n . Q u ery E x ec u t i o n . e a g er l y E x ec u t e C o mman d s (Q u ery E x ec u t i o n . sc a l a : 94) a t or g . a p a c h e . s p a r k . s ql . e x ec u t i o n . Q u ery E x ec u t i o n . co mman d E x ec u t e d$ lzycompute(QueryExecution.scala:81)
at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:79)
at org.apache.spark.sql.Dataset.(Dataset.scala:220)
at org.apache.spark.sql.Dataset $.$ anonfun$ofRows $2 (D a t a se t . sc a l a : 100) a t or g . a p a c h e . s p a r k . s ql . Sp a r k S ess i o n . w i t h A c t i v e (Sp a r k S ess i o n . sc a l a : 779) a t or g . a p a c h e . s p a r k . s ql . D a t a se t$ .ofRows(Dataset.scala:97)
at org.apache.spark.sql.SparkSession. $an o n f u n$ sql$1(SparkSession.scala:622)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:779)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:617)
at inputandoutput.Test05_Hive.main(Test05_Hive.java:27)

原因分析：

看报的异常是org.apache.hadoop.hive.ql.metadata.HiveException，说明是我Hive的metastore服务的问题，应该是启动，然后我又因为是在本机连接虚拟机中的Hive，所以应该还要开启Hiveserver2这个服务

解决方案：

开启metastore服务和Hiveserver2服务即可
开启脚本：

vim hiveservices.sh

插入如下内容：

#!/bin/bashHIVE_LOG_DIR=$HIVE_HOME/logs
if [ ! -d $HIVE_LOG_DIR ]
thenmkdir -p $HIVE_LOG_DIR
fi#检查进程是否运行正常，参数1为进程名，参数2为进程端口
function check_process()
{pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}')ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1)echo $pid[[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1
}function hive_start()
{metapid=$(check_process HiveMetastore 9083)cmd="nohup hive --service metastore >$HIVE_LOG_DIR/metastore.log 2>&1 &"[ -z "$metapid" ] && eval $cmd || echo "Metastroe服务已启动"server2pid=$(check_process HiveServer2 10000)cmd="nohup hive --service hiveserver2 >$HIVE_LOG_DIR/hiveServer2.log 2>&1 &"[ -z "$server2pid" ] && eval $cmd || echo "HiveServer2服务已启动"
}function hive_stop()
{
metapid=$(check_process HiveMetastore 9083)[ "$metapid" ] && kill $metapid || echo "Metastore服务未启动"server2pid=$(check_process HiveServer2 10000)[ "$server2pid" ] && kill $server2pid || echo "HiveServer2服务未启动"
}case $1 in
"start")hive_start;;
"stop")hive_stop;;
"restart")hive_stopsleep 2hive_start;;
"status")check_process HiveMetastore 9083 >/dev/null && echo "Metastore服务运行正常" || echo "Metastore服务运行异常"check_process HiveServer2 10000 >/dev/null && echo "HiveServer2服务运行正常" || echo "HiveServer2服务运行异常";;
*)echo Invalid Args!echo 'Usage: '$(basename $0)' start|stop|restart|status';;
esac