Kerberos、Impala、Druid的配合使用
|Word count:1.9k|Reading time:11min|Post View:
本篇介绍了如何在开启Kerberos认证的情况下,数据库连接池Druid和Impala的配合使用。
背景
由于生产环境开启了Kerberos的认证限制,原来可直接使用Druid连接池的方式便不可用了,需要重写Druid获取连接的逻辑,在其创建Impala连接时加入Kerberos认证的功能。
实现方式
ImpalaDruidDataSource.class1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
| import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.druid.pool.DruidPooledConnection;
import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException; import java.security.PrivilegedAction; import java.sql.SQLException;
@Slf4j public class ImpalaDruidDataSource extends DruidDataSource {
@Getter @Setter private String authUser;
@Getter @Setter private String impalaDriver;
@Getter @Setter private String krb5Realm;
@Getter @Setter private String krb5Kdc;
public DruidPooledConnection superGetConnection(long maxWaitMillis) throws SQLException { return super.getConnection(maxWaitMillis); }
public void supperInit() throws SQLException { super.init(); }
@Override public void init() throws SQLException { if (super.inited) { return; } String keytabFile = System.getProperty("krb5.keytab.conf"); String krb5File = System.getProperty("java.security.krb5.conf"); if (StringUtils.isBlank(keytabFile) || StringUtils.isBlank(krb5File)) { log.error("-Dkrb5.keytab.conf Or -Djava.security.krb5.conf not exist"); } System.setProperty("java.security.krb5.realm", krb5Realm); System.setProperty("java.security.krb5.kdc", krb5Kdc); System.setProperty("java.security.krb5.conf", krb5File); System.setProperty("hadoop.security.authentication", "kerberos"); Configuration conf = new Configuration(); UserGroupInformation.setConfiguration(conf); UserGroupInformation loginUser = null; try { UserGroupInformation.loginUserFromKeytab(authUser, keytabFile); loginUser = UserGroupInformation.getLoginUser(); } catch (IOException e) { e.printStackTrace(); } inited = false;
ImpalaDruidDataSource _this = this; assert loginUser != null; loginUser.doAs((PrivilegedAction<Void>) () -> { try { _this.supperInit(); } catch (Exception e) { e.printStackTrace(); } return null; }); }
@Override public DruidPooledConnection getConnection(final long maxWaitMillis) throws SQLException { UserGroupInformation loginUser = null; try { loginUser = UserGroupInformation.getLoginUser(); loginUser.checkTGTAndReloginFromKeytab(); } catch (IOException e) { e.printStackTrace(); log.error("kerberos error", e); } DruidPooledConnection conn = null; log.info(" [ Start to Connect Impala ] "); try { assert loginUser != null; conn = loginUser.doAs((PrivilegedAction<DruidPooledConnection>) () -> { DruidPooledConnection tcon = null; try { tcon = superGetConnection(maxWaitMillis); } catch (SQLException e) { e.printStackTrace(); } return tcon; }); log.info(" [ Connect Impala Success ] "); } catch (Exception e) { e.printStackTrace(); } return conn; }
}
|
依赖包:
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45
| <dependency> <groupId>com.cloudera.impala.jdbc</groupId> <artifactId>ImpalaJDBC41</artifactId> <version>${impala.jdbc.version}</version> <scope>system</scope> <systemPath>${pom.basedir}/libs/ImpalaJDBC41.jar</systemPath> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>${hadoop.version}</version> <exclusions> <exclusion> <artifactId>commons-logging</artifactId> <groupId>commons-logging</groupId> </exclusion> <exclusion> <artifactId>slf4j-api</artifactId> <groupId>org.slf4j</groupId> </exclusion> </exclusions> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-auth</artifactId> <version>${hadoop.version}</version> <exclusions> <exclusion> <artifactId>jsr305</artifactId> <groupId>com.google.code.findbugs</groupId> </exclusion> <exclusion> <artifactId>log4j</artifactId> <groupId>log4j</groupId> </exclusion> <exclusion> <artifactId>slf4j-log4j12</artifactId> <groupId>org.slf4j</groupId> </exclusion> <exclusion> <artifactId>slf4j-api</artifactId> <groupId>org.slf4j</groupId> </exclusion> </exclusions> </dependency>
|
使用方法
SpringMvc
applicationContext.xml
applicationContext.xml1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
| <bean name="impalaDataSource" class="cn.net.dev.analysis.impala.ImpalaDruidDataSource" init-method="init" destroy-method="close"> <property name="url" value="${impala.url}" /> <property name="username" value="${impala.username}" /> <property name="password" value="${impala.password}" /> <property name="driverClassName" value="${impala.jdbc.driver}"/> <property name="initialSize" value="3" /> <property name="minIdle" value="8" /> <property name="maxWait" value="500000" /> <property name="testWhileIdle" value="true" /> <property name="testOnBorrow" value="false" /> <property name="testOnReturn" value="false" /> <property name="timeBetweenEvictionRunsMillis" value="50000" /> <property name="minEvictableIdleTimeMillis" value="25200000" /> <property name="removeAbandoned" value="true" /> <property name="removeAbandonedTimeout" value="600" /> <property name="logAbandoned" value="true" /> <property name="krb5Realm" value="${impala.krb5.realm}"/> <property name="krb5Kdc" value="${impala.krb5.kdc}"/> <property name="authUser" value="${impala.kerberos.username}"/> <property name="impalaDriver" value="${impala.jdbc.driver}"/> <property name="maxActive" value="20" /> <property name="validationQuery" value="select 'Check'" /> </bean>
|
application.properties
application.properties1 2 3 4 5 6 7 8 9 10 11 12 13
|
impala.url=jdbc:impala://impala.proxy.example.com:21050;AuthMech=1;KrbRealm=EXAMPLE.COM;KrbHostFQDN=impala.proxy.example.com;KrbServiceName=impala impala.jdbc.driver=com.cloudera.impala.jdbc41.Driver
impala.username=etl impala.password=etl impala.krb5.realm=EXAMPLE.COM impala.krb5.kdc=ad01.example.com:88 impala.kerberos.username=[email protected]
|
启动时,在vm options
中输入如下:
1
| -Dspring.profiles.active=dev -Dsun.security.krb5.debug=false -Djava.security.krb5.conf=/Users/liyang/Documents/kerberos/krb5.conf -Dkrb5.keytab.conf=/Users/liyang/Documents/kerberos/etl.keytab
|
SpringBoot
DataSourceProperties.java
DataSourceProperties.java1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
| import java.util.Map;
import org.springframework.boot.context.properties.ConfigurationProperties;
import lombok.Data;
@ConfigurationProperties(prefix = DataSourceProperties.DS, ignoreUnknownFields = false) @Data public class DataSourceProperties { final static String DS = "spring.datasource";
private Map<String,String> mysql;
private Map<String,String> impala;
private Map<String,String> common;
}
|
DataSourceCommonProperties.java
DataSourceCommonProperties.java1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
| import org.springframework.boot.context.properties.ConfigurationProperties;
import lombok.Data;
@ConfigurationProperties(prefix = DataSourceCommonProperties.DS, ignoreUnknownFields = false) @Data public class DataSourceCommonProperties { final static String DS = "spring.datasource.common";
private int initialSize = 10; private int minIdle; private int maxIdle; private int maxActive; private int maxWait; private int timeBetweenEvictionRunsMillis; private int minEvictableIdleTimeMillis; private String validationQuery; private boolean testWhileIdle; private boolean testOnBorrow; private boolean testOnReturn; private boolean poolPreparedStatements; private int maxOpenPreparedStatements; private String filters; private String mapperLocations; private String typeAliasPackage; }
|
ImpalaDataSourceConfig.java
ImpalaDataSourceConfig.java1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97
| import java.sql.SQLException;
import javax.sql.DataSource;
import org.apache.ibatis.session.SqlSessionFactory; import org.mybatis.spring.SqlSessionFactoryBean; import org.mybatis.spring.annotation.MapperScan; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import cn.net.dev.analysis.core.pool.ImpalaDruidDataSource;
import lombok.extern.slf4j.Slf4j;
@Slf4j @Configuration @EnableConfigurationProperties({DataSourceProperties.class, DataSourceCommonProperties.class}) @MapperScan(basePackages = ImpalaDataSourceConfig.PACKAGE, sqlSessionFactoryRef = "impalaSqlSessionFactory") public class ImpalaDataSourceConfig {
static final String PACKAGE = "cn.net.dev.analysis.mapper.impala";
private static final String MAPPER_LOCATION = "classpath:mapper/impala/**/*.xml";
@Autowired private DataSourceCommonProperties dataSourceCommonProperties;
@Autowired private DataSourceProperties dataSourceProperties;
@Primary @Bean("impalaDruidDataSource") public DataSource dataSource() { ImpalaDruidDataSource dataSource = new ImpalaDruidDataSource(); dataSource.setUrl(dataSourceProperties.getImpala().get("url")); dataSource.setUsername(dataSourceProperties.getImpala().get("username")); dataSource.setPassword(dataSourceProperties.getImpala().get("password")); dataSource.setDriverClassName(dataSourceProperties.getImpala().get("driver-class-name")); dataSource.setAuthUser(dataSourceProperties.getImpala().get("auth-user")); dataSource.setImpalaDriver(dataSourceProperties.getImpala().get("driver-class-name")); dataSource.setKrb5Kdc(dataSourceProperties.getImpala().get("kerberos-kdc")); dataSource.setKrb5Realm(dataSourceProperties.getImpala().get("kerberos-realm")); dataSource.setInitialSize(dataSourceCommonProperties.getInitialSize()); dataSource.setMinIdle(dataSourceCommonProperties.getMinIdle()); dataSource.setMaxActive(dataSourceCommonProperties.getMaxActive()); dataSource.setMaxWait(dataSourceCommonProperties.getMaxWait()); dataSource.setTimeBetweenEvictionRunsMillis(dataSourceCommonProperties.getTimeBetweenEvictionRunsMillis()); dataSource.setMinEvictableIdleTimeMillis(dataSourceCommonProperties.getMinEvictableIdleTimeMillis()); dataSource.setValidationQuery(dataSourceCommonProperties.getValidationQuery()); dataSource.setTestWhileIdle(dataSourceCommonProperties.isTestWhileIdle()); dataSource.setTestOnBorrow(dataSourceCommonProperties.isTestOnBorrow()); dataSource.setTestOnReturn(dataSourceCommonProperties.isTestOnReturn()); dataSource.setPoolPreparedStatements(dataSourceCommonProperties.isPoolPreparedStatements());
try { dataSource.setFilters(dataSourceCommonProperties.getFilters()); } catch (SQLException e) { log.error("Druid configuration initialization filter error.", e); } return dataSource; }
@Primary @Bean(name = "impalaTransactionManager") public DataSourceTransactionManager primaryTransactionManager() throws SQLException { return new DataSourceTransactionManager(dataSource()); }
@Primary @Bean(name = "impalaSqlSessionFactory") public SqlSessionFactory primarySqlSessionFactory(@Qualifier("impalaDruidDataSource") DataSource primaryDataSource) throws Exception { final SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean(); sessionFactory.setDataSource(primaryDataSource); sessionFactory.setMapperLocations(new PathMatchingResourcePatternResolver() .getResources(ImpalaDataSourceConfig.MAPPER_LOCATION));
return sessionFactory.getObject(); } }
|
application.yml
application.yml1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
| spring: datasource: mysql: type: com.alibaba.druid.pool.DruidDataSource url: jdbc:mysql://10.10.20.22:3306/test?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull username: root password: 123456 driver-class-name: com.mysql.cj.jdbc.Driver impala: url: jdbc:impala://impala.proxy.example.com:21051;AuthMech=1;KrbRealm=EXAMPLE.COM;KrbHostFQDN=impala.proxy.example.com;KrbServiceName=impala type: cn.net.dev.analysis.core.pool.ImpalaDruidDataSource username: etl password: etl driver-class-name: com.cloudera.impala.jdbc41.Driver auth-user: [email protected] kerberos-kdc: ad01.example.com:88 kerberos-realm: EXAMPLE.COM common: initialSize: 1 minIdle: 1 maxIdle: 5 maxActive: 50 maxWait: 10000 timeBetweenEvictionRunsMillis: 10000 minEvictableIdleTimeMillis: 300000 validationQuery: select 'x' testWhileIdle: true testOnBorrow: false testOnReturn: false poolPreparedStatements: true maxOpenPreparedStatements: 20 filters: stat
|
启动vm options
中填写如下内容:
1
| -Dsun.security.krb5.debug=false -Djava.security.krb5.conf=/Users/liyang/Documents/kerberos/krb5.conf -Dkrb5.keytab.conf=/Users/liyang/Documents/kerberos/etl.keytab
|