Java 连接 Kerberos认证下的Spark Thrift Server/Hive Server总结

2021-05-10

前些天发现了一个巨牛的人工智能学习网站，通俗易懂，风趣幽默，忍不住给大家分享一下。点击跳转到网站：https://www.captainai.net/dongkelun

欢迎关注我的公众号

前言

总结Java如何连接Kerberos认证下的Spark Thrift Server/Hive Server总结

启动

关于如何启动 Spark Thrift Server和 Hive Server 请参考https://dongkelun.com/2021/02/19/javaSparkThriftServer/

Java 代码

pom 依赖

<dependencies>
    <dependency>
        <groupId>org.apache.hive</groupId>
        <artifactId>hive-jdbc</artifactId>
        <version>1.2.1</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>2.6.5</version>
    </dependency>
</dependencies>

关于依赖版本对应同样请参考https://dongkelun.com/2021/02/19/javaSparkThriftServer/

配置文件

hive.service.keytab和krb5.conf均为kerberos认证相关，从服务器上下载，krb5.conf路径在/etc下，具体根据自己服务器的配置去查找

krb5.conf

[libdefaults]
  renew_lifetime = 7d
  forwardable = true
  default_realm = INDATA.COM
  ticket_lifetime = 24h
  dns_lookup_realm = false
  dns_lookup_kdc = false
  default_ccache_name = /tmp/krb5cc_%{uid}
  #default_tgs_enctypes = aes des3-cbc-sha1 rc4 des-cbc-md5
  #default_tkt_enctypes = aes des3-cbc-sha1 rc4 des-cbc-md5

[domain_realm]
  indata.com = INDATA.COM

[logging]
  default = FILE:/var/log/krb5kdc.log
  admin_server = FILE:/var/log/kadmind.log
  kdc = FILE:/var/log/krb5kdc.log

[realms]
  INDATA.COM = {
    admin_server = indata-192.168.44.128.indata.com:17490
    kdc = indata-192.168.44.128.indata.com
    kdc = indata-192.168.44.129.indata.com
  }

代码

package com.dkl.blog;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;

import java.io.IOException;
import java.sql.*;

/**
 * Created by dongkelun on 2021/5/7 10:06
 *
 * java 访问 kerberos认证下的 Hive Server和 Spark Thrift Server
 */
public class SparkThriftServerDemoWithKerberos {
    private static String HIVE_JDBC_URL = "jdbc:hive2://192.168.44.128:10000/sjtt;principal=hive/indata-192-168-44-128.indata.com@INDATA.COM";
    private static final String SPARK_JDBC_URL = "jdbc:hive2://192.168.44.128:20003/sjtt;" +
            "principal=HTTP/indata-192-168-44-128.indata.com@INDATA.COM?" +
            "hive.server2.transport.mode=http;hive.server2.thrift.http.path=cliservice;";
    private static final String PRINCIPAL = "hive/indata-192-168-44-128.indata.com@INDATA.COM";
    private static final String KEYTAB = "D:\\conf\\inspur\\hive.service.keytab";
    private static final String KRB5 = "D:\\conf\\inspur\\krb5.conf";

    private static Configuration conf = null;

    static {
        conf = new Configuration();
    }

    public static void main(String[] args) throws SQLException {
        loadConfiguration();
        //----------------------------------connect hive----------------------------------//
        System.out.println("select from hive");
        jdbcDemo(HIVE_JDBC_URL);

        //------------------------------connect spark thrift server-----------------------//
        System.out.println("select from spark thrift server");
        jdbcDemo(SPARK_JDBC_URL);
    }

    public static void jdbcDemo(String jdbc_url) throws SQLException {
        Connection connection = null;
        try {
            connection = DriverManager.getConnection(jdbc_url);
            selectTable(connection);
        } catch (SQLException e) {
            e.printStackTrace();
        } finally {
            connection.close();
        }
    }

    public static void selectTable(Connection connection) {
        String sql = "select * from trafficbase_cljbxx limit 10";
        Statement stmt = null;
        ResultSet rs = null;
        try {
            stmt = connection.createStatement();
            rs = stmt.executeQuery(sql);
            System.out.println("=====================================");
            while (rs.next()) {
                System.out.println(rs.getString(1) + "," + rs.getString(2));
            }
            System.out.println("=====================================");
        } catch (SQLException e) {
            e.printStackTrace();
        } finally {
            close(stmt);
            close(rs);
        }

    }

    private static void loadConfiguration() {
        // 初始化配置文件
        try {
            conf.set("hadoop.security.authentication", "kerberos");
            System.setProperty("java.security.krb5.conf", KRB5);// krb5文件路径
            UserGroupInformation.setConfiguration(conf);
            UserGroupInformation.loginUserFromKeytab(PRINCIPAL, KEYTAB);// 入参：principal、keytab文件
        } catch (IOException ioE) {
            System.err.println("使用keytab登陆失败");
            ioE.printStackTrace();
        }

    }

    /**
     * 关闭Statement
     *
     * @param stmt
     */
    private static void close(Statement stmt) {
        if (stmt != null) {
            try {
                stmt.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }

    /**
     * 关闭ResultSet
     *
     * @param rs
     */
    private static void close(ResultSet rs) {
        if (rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }
}