一、新建maven项目,编写pom.xml文件
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.enn</groupId>
<artifactId>testDB</artifactId>
<version>1.0-SNAPSHOT</version>
<!--配置属性和版本号 -->
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hadoop.version>2.6.0-cdh5.8.0</hadoop.version>
<hbase.version>1.2.0-cdh5.8.0</hbase.version>
</properties>
<!--配置依赖库地址(用于加载CDH依赖的jar包) -->
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
<dependencies>
<!-- hadoop依赖包 -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-runtime</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- hbase依赖包 -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- logback日志依赖包 -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>1.1.3</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.1.3</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-access</artifactId>
<version>1.1.3</version>
</dependency>
</dependencies>
<!-- 编译配置 -->
<build>
<plugins>
<!-- 编译插件 -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
<encoding>${project.build.sourceEncoding}</encoding>
</configuration>
</plugin>
</plugins>
</build>
</project>
二、编写配置文件:CreateTable.xml(注意:该文件需要放到resources目录下)
<root>
<column>
<MaxVersions>1</MaxVersions><!-- 最大版本数,默认为1 -->
<MinVersions>0</MinVersions><!-- 最小版本数,默认为0,如果大于0,必须设置TTL(数据保存时间) -->
<BlockSize>65536</BlockSize><!-- HFile存储块大小,默认64KB -->
<BlockCacheEnabled>true</BlockCacheEnabled><!-- 是否启用缓存,如果某些数据经常被查询则启用,缓存数据分为三级 -->
<InMemory>false</InMemory><!-- 列族数据是否使用内存模式,true则将表列数据写入硬盘且写入内存,会使用缓存的最高级别 -->
<TimeToLive>2147483647</TimeToLive><!--单位秒 生命周期 默认永久(2147483647s即68年) -->
<BloomFilterType>ROW</BloomFilterType><!-- 布隆过滤,用于查询,此处所有列族均使用该过滤,默认ROW是为了与本项目的get查询方法对应,可选NONE、ROWCOL -->
<CompressionType>LZ4</CompressionType><!-- 压缩类型,支持LZ4、LZO、GZIP、SNAPPY压缩,LZO、SNAPPY需要集群安装,LZ4需要调用hadoop的lib,GZIP压缩率高但速度更慢 -->
<CacheBloomsOnWrite>false</CacheBloomsOnWrite><!-- 写布隆过滤时是否缓存 -->
<CacheDataOnWrite>false</CacheDataOnWrite><!-- 写数据时是否缓存-->
<CacheIndexesOnWrite>false</CacheIndexesOnWrite><!-- 写索引时是否缓存 -->
<CompressTags>true</CompressTags><!-- 是否压缩HFile的标签 -->
<Scope>1</Scope><!-- 注意:本参数有待考证:范围标签,0则集群间复制时不允许,1则允许 -->
<KeepDeletedCells>false</KeepDeletedCells><!-- 是否保留删除的单元(true则在TTl前仍可以get或scan) -->
<DataBlockEncoding>PREFIX_TREE</DataBlockEncoding><!-- 数据编码,优化重复数据的存储,能够节省空间 -->
</column>
<table>
<FamilyName>f</FamilyName><!-- 列族,用逗号分割,尽量只使用一个列族,多列族会导致分割合并、查询效率低下 -->
<TableName>greatgas:enn_test2222</TableName><!-- 表名,多表以逗号分开 -->
<RegionNum>32</RegionNum><!-- 预分区数 -->
</table>
<zookeeper>
<Quorum>slave-29.dev.cluster.enn.cn:2181,slave-30.dev.cluster.enn.cn:2181,slave-31.dev.cluster.enn.cn:2181</Quorum><!-- zookeeper主机 -->
<Parent>/hbase</Parent><!-- hbase在zookeeper上的存储目录,可以称作是库,多个库以逗号分开 -->
</zookeeper>
</root>
三、编写工具类:HBaseConf.java
package createTable;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.XMLConfiguration;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.URL;
import java.util.NoSuchElementException;
/**
* 获取创建表配置文件的内容
*/
public class HBaseConf {
Logger logger = LoggerFactory.getLogger(HBaseConf.class);
//列属性配置
int MaxVersions;//最大版本数
int MinVersions;//最小版本数
int Blocksize;//块大小
boolean BlockCacheEnabled;//块是否可缓存
boolean InMemory;//
int TimeToLive;//存储时长
BloomType BloomFilterType;//存储HFile过滤器
Algorithm CompressionType;//压缩类型
boolean CacheBloomsOnWrite;//
boolean CacheDataOnWrite;//
boolean CacheIndexesOnWrite;//
String CompressTags;//
int Scope;//
boolean KeepDeletedCells;//是否保存删除的数据
DataBlockEncoding DataBlockEncoding;//
//表属性配置
String FamilyName;//列族名
String TableName;//表名
String RegionNum;//分区数
//hbase连接配置
String Quorum;//zookeeper主机
String Parent;//zookeeper存储目录
public HBaseConf readXml() {
try {
XMLConfiguration xmlConfiguration = getXml();
xmlConfiguration.setThrowExceptionOnMissing(true);//获取不到指定属性则报错
this.MaxVersions = xmlConfiguration.getInt("column.MaxVersions");
this.MinVersions = xmlConfiguration.getInt("column.MinVersions");
this.Blocksize = xmlConfiguration.getInt("column.BlockSize");
this.BlockCacheEnabled = xmlConfiguration.getBoolean("column.BlockCacheEnabled");
this.InMemory = xmlConfiguration.getBoolean("column.InMemory");
this.TimeToLive = xmlConfiguration.getInt("column.TimeToLive");
this.BloomFilterType = BloomType.valueOf(xmlConfiguration.getString("column.BloomFilterType").toUpperCase());
this.CompressionType = Algorithm.valueOf(xmlConfiguration.getString("column.CompressionType").toUpperCase());
this.CacheBloomsOnWrite = xmlConfiguration.getBoolean("column.CacheBloomsOnWrite");
this.CacheDataOnWrite = xmlConfiguration.getBoolean("column.CacheDataOnWrite");
this.CacheIndexesOnWrite = xmlConfiguration.getBoolean("column.CacheIndexesOnWrite");
this.CompressTags = xmlConfiguration.getString("column.CompressTags");
this.Scope = xmlConfiguration.getInt("column.Scope");
this.KeepDeletedCells = xmlConfiguration.getBoolean("column.KeepDeletedCells");
this.DataBlockEncoding = DataBlockEncoding.valueOf(xmlConfiguration.getString("column.DataBlockEncoding").toUpperCase());
this.FamilyName = xmlConfiguration.getString("table.FamilyName");
this.TableName = xmlConfiguration.getString("table.TableName");
this.RegionNum = xmlConfiguration.getString("table.RegionNum");
this.Quorum = xmlConfiguration.getString("zookeeper.Quorum");
this.Parent = xmlConfiguration.getString("zookeeper.Parent");
return this;
} catch (ConfigurationException ex) {
logger.error("创建xml解读类出现错误");
ex.printStackTrace();
return null;
} catch (NoSuchElementException ex) {
logger.error("读取'建表的参数配置文件'出现错误,请确保已配置如下参数:{}.", ex.getMessage());
ex.printStackTrace();
return null;
}
}
/**
* 读取默认的建表配置文件
*
* @return 配置文件
* @throws ConfigurationException
*/
private XMLConfiguration getXml() throws ConfigurationException {
URL url = HBaseConf.class.getClassLoader().getResource("CreateTable.xml");
XMLConfiguration xmlConfiguration=new XMLConfiguration();
xmlConfiguration.setDelimiterParsingDisabled(true);
xmlConfiguration.setAttributeSplittingDisabled(true);
xmlConfiguration.load(url);
return xmlConfiguration;
}
/**
* 读取指定文件
*
* @param file 指定的建表配置文件名
* @return 配置文件
* @throws ConfigurationException
*/
private XMLConfiguration getXml(String file) throws ConfigurationException {
XMLConfiguration xmlConfiguration=new XMLConfiguration();
xmlConfiguration.setDelimiterParsingDisabled(true);
xmlConfiguration.setAttributeSplittingDisabled(true);
xmlConfiguration.load(file);
return xmlConfiguration;
}
}
编写主类:CreateTable.java(注意:如果有kerberos认证,则需要2个文件---》krb5.conf和e_lvbin.keytab)
package createTable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
/**
* 创建Hbase分区表的实现
* 测试通过
*/
public class CreateTable {
static Logger logger = LoggerFactory.getLogger(CreateTable.class);//创建日志对象
public static void main(String[] args) {
HBaseConf conf = null;//创建配置文件类
conf = new HBaseConf().readXml();//获取配置文件的配置参数类
try {
for (String parent : conf.Parent.split(",")) {
for (String table : conf.TableName.split(",")) {
create(conf, parent, table);
}
}
} catch (Exception e) {
logger.error("建表失败!");
e.printStackTrace();
}
}
/**
* 建表
*
* @param conf 配置类
* @param parent 库名
* @param table 表名
* 作用: 创建分区表
* @throws IOException
*/
public static void create(HBaseConf conf, String parent, String table) throws IOException {
System.setProperty("java.security.krb5.conf","F:/krb5.conf"); //windows下 ,本地执行需要
Configuration configuration = HBaseConfiguration.create();//创建Hbase连接配置
configuration.set("hbase.zookeeper.quorum", "slave-31.dev.cluster.em.cn:2181");
configuration.set("hbase.rootdir", "hdfs://mycluster/hbase");
configuration.set("hadoop.security.authentication", "kerberos");
configuration.set("hbase.security.authentication", "kerberos");
configuration.set("hbase.security.authorization", "true");
configuration.set("hbase.master.kerberos.principal", "hbase/_HOST@EM.CN");
configuration.set("hbase.thrift.kerberos.principal", "hbase/_HOST@EM.CN");
configuration.set("hbase.regionserver.kerberos.principal", "hbase/_HOST@EM.CN");
configuration.set("hbase.zookeeper.property.clientPort", "2181");
configuration.set(TableInputFormat.INPUT_TABLE, table);
configuration.set("hbase.rpc.timeout", "10000");
configuration.set("hbase.client.retries.number", "5");
configuration.set("hbase.client.pause", "5000");
configuration.set("hbase.client.scanner.timeout.period", "50000");
String user = "e_lvbin@EM.CN";
String keyPath = "F:/e_lvbin.keytab"; //本地认证需要的文件
UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(user, keyPath);
HBaseAdmin hBaseAdmin = null;//创建表管理
try {
hBaseAdmin = new HBaseAdmin(configuration);
logger.info("hbase库是:{}............", parent);
if (hBaseAdmin.tableExists(table)) {
logger.error("{}表已经存在,无法创建!", table);
} else {
logger.info("正在创建表:{}..............", table);
HTableDescriptor hTableDescriptor = new HTableDescriptor(TableName.valueOf(table));//创建表描述器
//hTableDescriptor.setMaxFileSize();//集群配置中应该有,这些不设置应该也行
for (String familyName : conf.FamilyName.split(",")) {//针对不同的列族建立列描述器
HColumnDescriptor hd = new HColumnDescriptor(familyName);//创建列描述器
hd.setBlockCacheEnabled(conf.BlockCacheEnabled);
hd.setBlocksize(conf.Blocksize);
hd.setBloomFilterType(conf.BloomFilterType);
hd.setCacheBloomsOnWrite(conf.CacheBloomsOnWrite);
hd.setCacheDataOnWrite(conf.CacheDataOnWrite);
hd.setCacheIndexesOnWrite(conf.CacheIndexesOnWrite);
hd.setCompressionType(conf.CompressionType);
hd.setDataBlockEncoding(conf.DataBlockEncoding);
hd.setInMemory(conf.InMemory);
hd.setMaxVersions(conf.MaxVersions);
hd.setMinVersions(conf.MinVersions);
hd.setTimeToLive(conf.TimeToLive);
hd.setKeepDeletedCells(conf.KeepDeletedCells);
hd.setScope(conf.Scope);
hTableDescriptor.addFamily(hd);//表描述器加载该列描述器
}
//注意建立预分区的startKey与endKey类型要与你插入数据时的hash值对应起来,此处是short数,那个你插入时rowkey的前缀应该是short的byte数组而不是字符串的byte数组
hBaseAdmin.createTable(hTableDescriptor,
Bytes.toBytes((short)(0)), Bytes.toBytes((short)(0x7FFF)), Integer.parseInt(conf.RegionNum));//创建表-参数分别是:表描述、起始key、结束key、分区数
logger.info("表创建完成:{}!", table);
}
} catch (IOException ex) {
ex.printStackTrace();
} finally {
if (null != hBaseAdmin) {
try {
hBaseAdmin.close();
} catch (IOException e) {
logger.info("表连接关闭失败:{}!", table);
e.printStackTrace();
}
}
}
}
}
四、执行主类:CreateTable.java
五、xshell上查看是否创建分区表成功
注意:如果出现空指针异常,请将resources目录下的CreateTable.xml另起一个名称CreateTable222.xml,
即可解决!!
本文作者:网友 来源:网络收集
CIO之家 www.ciozj.com 微信公众号:imciow