hbase生成分区种子    1 package com.spdbccc.edm.storm.common;  2   3 import java.util.List;  4   5 import org.apache.commons.lang3.StringUtils;  6 import org.apache.hadoop.hbase.util.Bytes;  7   8 import com.google.common.collect.Lists;  9  10 /** 11  * HBase表管理器, 对Hbase表的rowkey规则生成,表名规则定义 12  *  13  * @author WJ 14  * 15  */ 16 public class HTableManager { 17  18     public static final byte[] DEFAULT_FAMILY_NAME = Bytes.toBytes("f1"); 19  20     private static final String[] PARTITIONS = generatPartitionSeed(); 21  22     /** 23      * 生成3844个分区种子 24      *  25      * @return String[] 26      */ 27     public static String[] generatPartitionSeed() { 28         List
     
       seeds = Lists.newArrayList(); 29         for (int i = '0'; i <= '9'; i++) { 30             seeds.add((char) i); 31         } 32         for (int i = 'A'; i <= 'Z'; i++) { 33             seeds.add((char) i); 34         } 35         for (int i = 'a'; i <= 'z'; i++) { 36             seeds.add((char) i); 37         } 38         int k = 0; 39         String[] partions = new String[seeds.size() * seeds.size()]; 40         for (int i = 0; i < seeds.size(); i++) { 41             for (int j = 0; j < seeds.size(); j++) { 42                 partions[k] = StringUtils.join(seeds.get(i), seeds.get(j)); 43                 k++; 44             } 45         } 46         return partions; 47     } 48  49     /** 50      * 按指定数量生成分区种子 51      *  52      * @param limit 53      * @return String[] 54      */ 55     public static String[] generatPartitionSeed(int limit) { 56         int size = PARTITIONS.length; 57         int[] space = new int[limit]; 58         for (int pt = 0; pt < size;) { 59             for (int j = 0; j < space.length; j++) { 60                 ++space[j]; 61                 pt++; 62                 if (pt == size) { 63                     break; 64                 } 65             } 66         } 67         String[] seed = new String[limit + 1]; 68         int position = 0; 69         for (int i = 0; i < space.length; i++) { 70             seed[i] = PARTITIONS[position]; 71             position += space[i]; 72         } 73         seed[seed.length - 1] = PARTITIONS[PARTITIONS.length - 1]; 74         return seed; 75     } 76  77     public static String generatRowkey(String str) { 78         int i = Math.abs(str.hashCode() % PARTITIONS.length); 79         return StringUtils.join(PARTITIONS[i], "-", str); 80     } 81  82     public static byte[] generatByteRowkey(String str) { 83         int i = Math.abs(str.hashCode() % PARTITIONS.length); 84         return Bytes.toBytes(StringUtils.join(PARTITIONS[i], "-", str)); 85     } 86  87     public static String getEventLogTableName(String event) { 88         return StringUtils.join("EVENT_LOG_", event); 89     } 90      91     public static String getGroupVarTableName() { 92         return "CUSTOM_VARIABLE_GROUP"; 93     } 94  95      96     public static String getActivityVarTableName() { 97         return "CUSTOM_VARIABLE_ACTIVITY"; 98     } 99 100     public static String getUserVarTableName() {101         return "CUSTOM_VARIABLE_USER";102     }103 104     public static String getUserInfoTableName() {105         return "USER_WIDE_PUB";106     }107 108     public static String getCardInfoTableName() {109         return "CARD_WIDE_PUB";110     }111 112     public static String getAcctInfoTableName() {113         return "ACCT_WIDE_PUB";114     }115 116     public static String getMetricTableName() {117         return "METRICS";118     }119 120     public static String getCustDefinitionTableName() {121         return "CUST_USER_DEFINITION";122     }123 124     public static String getCardDefinitionTableName() {125         return "CARD_USER_DEFINITION";126     }127     128     public static String getEventLogLbsHisName(){129         return "EVENT_LOG_LBS_HIS";130     }131     132     public static String getRankLogName(){133         return "RANK_LOG";134     }135 136      public static void main(String[] args) {137      String[] arr = generatPartitionSeed(101);138      for (int i = 0; i < arr.length; i++) {139      System.out.println(arr[i]);140      }141      }142 }
     

 

按指定分区数量创建预分区表  1 package com.spdbccc.edm.storm.common; 2  3 import org.apache.hadoop.conf.Configuration; 4 import org.apache.hadoop.hbase.HBaseConfiguration; 5 import org.apache.hadoop.hbase.HColumnDescriptor; 6 import org.apache.hadoop.hbase.HTableDescriptor; 7 import org.apache.hadoop.hbase.TableName; 8 import org.apache.hadoop.hbase.client.HBaseAdmin; 9 import org.apache.hadoop.hbase.io.compress.Compression;10 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;11 import org.apache.hadoop.hbase.util.Bytes;12 13 public class HTableUtil {14 15     private static final int MAX_FILE_SIZE = 1024 * 1024 * 256;16 17     public static void main(String[] args) throws Exception {18         int limit = 100;19 20         createHBaseTable(HTableManager.getEventLogTableName("TEST"), limit);21 22         // createHBaseTable(HTableManager.getEventLogTableName("DH"), limit);23 24         // createHBaseTable(HTableManager.getActivityVarTableName(), limit);25 26         // createHBaseTable(HTableManager.getUserVarTableName(), limit);27 28     }29 30     private static HTableDescriptor getHTableDescriptor(String tableName) {31         HColumnDescriptor columnDescriptor = new HColumnDescriptor(HTableManager.DEFAULT_FAMILY_NAME);32         columnDescriptor.setCompressionType(Compression.Algorithm.SNAPPY);33         columnDescriptor.setCompactionCompressionType(Compression.Algorithm.SNAPPY);34         // columnDescriptor.setTimeToLive(60 * 60 * 24 * 365 * 1);35         columnDescriptor.setBlockCacheEnabled(true);36         columnDescriptor.setDataBlockEncoding(DataBlockEncoding.NONE);37 38         HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));39         desc.setCompactionEnabled(true);40         desc.setMaxFileSize(MAX_FILE_SIZE);41         desc.addFamily(columnDescriptor);42         return desc;43     }44 45     /**46      * 创建表47      * 48      * @param tableName49      *            表名50      * @param partitionSeedLimit51      *            预分区数量，最大数量384452      * @throws Exception53      */54     public static void createHBaseTable(String tableName, int partitionSeedLimit) throws Exception {55         if (partitionSeedLimit > 3844 || partitionSeedLimit < 1) {56             throw new IllegalArgumentException("PartitionSeedLimit must be > 0 and < 3844.");57         }58         System.out.println("init HBase admin...");59         Configuration conf = HBaseConfiguration.create();60         HBaseAdmin admin = new HBaseAdmin(conf);61 62         if (admin.tableExists(tableName)) {63             if (!admin.isTableDisabled(tableName)) {64                 admin.disableTable(tableName);65             }66             admin.deleteTable(tableName);67             System.out.println(String.format("Table is exist, drop table %s successed.", tableName));68         }69 70         System.out.println(String.format("Creating HBase table %s, partition seed limit %d.", tableName, partitionSeedLimit));71         admin.createTable(getHTableDescriptor(tableName), Bytes.toByteArrays(HTableManager.generatPartitionSeed(partitionSeedLimit)));72         System.out.println(String.format("HBase table %s is created.", tableName));73         admin.close();74         System.out.println("==============================================");75     }76 77     // public static void main(String[] args) throws Exception {78     // String tableName = HTableManager.getEventLogTableName(args[0]);79     // int limit = Integer.parseInt(args[1]);80     // System.out.println("创建表: " + tableName);81     // createHBaseTable(tableName, limit);82     // System.out.println("创建成功: " + tableName);83     // }84 85 }

 

生成符合规则的rowkey 1 String key = "NE123456789";2 String messageKey = HTableManager.generatRowkey(key) + "-" + YZYT_msg_type;