問題及背景：同一用戶的三家公司的物理集群合并，合并后用dataspace+kerberos控制不同公司對集群資料的訪問權限，三家公司分別使用獨立的kerberos票據訪問，特定的namespace,而生產環境部署的kylin-2.0/2.1只能保存cuboid到hbase 的 default namespace。

解決的過程及方案：
1、經多方查找發現， Kylin源碼github項目，最新的master分支已經支持更改存儲的hbase的默認namespace，但是官方對外發布的最新安裝包才更新到Kylin-2.1版本，且最新源碼并沒有經過生產的驗證，不敢冒然部署，于是想辦法在現有的版本上重新編譯源碼、打包。
2、經多方搜索確認更改hbase namespace的commit是kylin-2846，于是查找相關實現的細節。

解決思路：
1、首先按照github上提供的修改方法更改了https://github.com/apache/kylin/commit/ffcbdb7f4288fa6011f018163bdce9602415cf10
中的五個類，重新編譯打包，執行到convert cuboid Data to HFile這一步的時候拋出異常，找不到Hbase表

org.apache.hadoop.hbase.TableNotFoundException:ISW:KYLIN_F3KD3ZC1LU at org.apache.hadoop.hbase.client.HBaseAdmin.getTableDescriptor(HBaseAdmin.java:575)

而這時，我在hbase shell指定的namespace查看，已經生成了hbase表，為什么會找不到呢？繼續查找原因
2、繼續找Kylin-2846相關的內容，發現有人遇到同樣的問題，地址：https://issues.apache.org/jira/browse/KYLIN-2846
原來github上給出的代碼少了很關鍵的一步：

---.../main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java | 2 +-1 file changed, 1 insertion(+), 1 deletion(-)diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java index 9f9b382..7a583ca 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java @@ -92,7 +92,7 @@ public class CubeHFileJob extends AbstractHadoopJob {attachCubeMetadata(cube, job.getConfiguration());Configuration hbaseConf = HBaseConfiguration.create(getConf()); - HTable htable = new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME).toUpperCase()); + HTable htable = new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME));// Automatic config !HFileOutputFormat.configureIncrementalLoad(job, htable); -- 1.9.1 具體的修改步驟如下：From 004e1bcb983442a1a2c69f7d1d6c09dc29d4301e Mon Sep 17 00:00:00 2001 From: Liu Shaohui <liushaohui@xiaomi.com> Date: Tue, 5 Sep 2017 15:50:29 +0800 Subject: [PATCH] Add a config of hbase namespace for cube storage---.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++core-common/src/main/resources/kylin-defaults.properties | 3 +++core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java | 2 ++.../main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java | 2 +-.../main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java | 2 +-5 files changed, 11 insertions(+), 2 deletions(-)diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 749b515..3f53023 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -712,6 +712,10 @@ abstract public class KylinConfigBase implements Serializable {return StorageURL.valueOf(url);}+ public String getHBaseStorageNameSpace() { + return getOptional("kylin.storage.hbase.namespace", "default"); + } +public String getHBaseClusterFs() {return getOptional("kylin.storage.hbase.cluster-fs", "");} diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties index a3277ce..34fb5a0 100644 --- a/core-common/src/main/resources/kylin-defaults.properties +++ b/core-common/src/main/resources/kylin-defaults.properties @@ -65,6 +65,9 @@ kylin.source.hive.redistribute-flat-table=true# The storage for final cube file in hbasekylin.storage.url=hbase+# The namespace for hbase storage +kylin.storage.hbase.namespace=default +# Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4]kylin.storage.hbase.compression-codec=nonediff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 043993c..484b977 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -684,11 +684,13 @@ public class CubeManager implements IRealizationProvider {}private String generateStorageLocation() { + String namespace = config.getHBaseStorageNameSpace();String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix;String tableName = "";Random ran = new Random();do {StringBuffer sb = new StringBuffer(); + sb.append(namespace).append(":");sb.append(namePrefix);for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length()))); diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java index 30616c5..d1caccb 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java @@ -47,7 +47,7 @@ public class BulkLoadJob extends AbstractHadoopJob {options.addOption(OPTION_CUBE_NAME);parseOptions(options, args);- String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase(); + String tableName = getOptionValue(OPTION_HTABLE_NAME);// e.g// /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/// end with "/" diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java index 1a624c4..9f9b382 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java @@ -68,7 +68,7 @@ public class CubeHFileJob extends AbstractHadoopJob {Path partitionFilePath = new Path(getOptionValue(OPTION_PARTITION_FILE_PATH));Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH)); - String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase(); + String cubeName = getOptionValue(OPTION_CUBE_NAME);CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());-- 1.9.1 2、、、From 677400ee7b4cf5be0ddc9b595d80e02a0ba87c12 Mon Sep 17 00:00:00 2001 From: Liu Shaohui <liushaohui@xiaomi.com> Date: Fri, 15 Sep 2017 14:34:18 +0800 Subject: [PATCH] Addium for KYLIN-2846---.../main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java | 2 +-1 file changed, 1 insertion(+), 1 deletion(-)diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java index 9f9b382..7a583ca 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java @@ -92,7 +92,7 @@ public class CubeHFileJob extends AbstractHadoopJob {attachCubeMetadata(cube, job.getConfiguration());Configuration hbaseConf = HBaseConfiguration.create(getConf()); - HTable htable = new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME).toUpperCase()); + HTable htable = new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME));// Automatic config !HFileOutputFormat.configureIncrementalLoad(job, htable); -- 1.9.1 3、clean htablediff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java index 3728ea1..2d95542 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java +++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java @@ -46,15 +46,28 @@ public class StorageCleanJobHbaseUtil {public static void cleanUnusedHBaseTables(boolean delete, int deleteTimeout) throws IOException {Configuration conf = HBaseConfiguration.create(); - CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); + KylinConfig config = KylinConfig.getInstanceFromEnv(); + CubeManager cubeMgr = CubeManager.getInstance(config);// get all kylin hbase tablestry (HBaseAdmin hbaseAdmin = new HBaseAdmin(conf)) { - String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix; + String namespace = config.getHBaseStorageNameSpace(); + StringBuffer sb = new StringBuffer(); + String tableNamePrefix = null; + if(namespace.equals("default") || namespace.equals("")){ + tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix; + }else{ + sb.append(config.getHBaseStorageNameSpace()).append(":"); + sb.append(IRealizationConstants.SharedHbaseStorageLocationPrefix); + tableNamePrefix = sb.toString(); + } +HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*"); +List<String> allTablesNeedToBeDropped = new ArrayList<String>();for (HTableDescriptor desc : tableDescriptors) { +String host = desc.getValue(IRealizationConstants.HTableTag); - if (KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix().equalsIgnoreCase(host)) { + if (config.getMetadataUrlPrefix().equalsIgnoreCase(host)) {//only take care htables that belongs to self, and created more than 2 daysallTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());}

修改完后，core-common, core-cure, storage-hbase三個目錄重新打jar包，覆蓋原Kylin安裝目錄KYLIN_HOME/tomcat/webapps/kylin.war下對應的jar包，重啟Kylin，問題解決。

總結

以上是生活随笔為你收集整理的Kylin修改默认hbase namespace命名空间default的解决方案的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。