[#572] [Backport] increase dispersion of row key for SqlMetaData table

collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseSqlMetaDataCompatibility.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2014 NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.navercorp.pinpoint.collector.dao.hbase;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Repository;
+
+import com.navercorp.pinpoint.collector.dao.SqlMetaDataDao;
+import com.navercorp.pinpoint.common.hbase.HBaseAdminTemplate;
+import com.navercorp.pinpoint.common.hbase.HBaseTables;
+import com.navercorp.pinpoint.thrift.dto.TSqlMetaData;
+
+/**
+ * @author minwoo.jung
+ */
+//@Repository
+public class HbaseSqlMetaDataCompatibility implements SqlMetaDataDao {
+
+    private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+    private final boolean SQL_METADATA_VER2_EXISTED;
+    private final boolean SQL_METADATA_EXISTED;
+
+    @Autowired
+    private SqlMetaDataDao hbaseSqlMetaDataPastVersionDao;
+
+    @Autowired
+    private SqlMetaDataDao hbaseSqlMetaDataDao;
+
+    @Autowired
+    public HbaseSqlMetaDataCompatibility(HBaseAdminTemplate hBaseAdminTemplate) {
+        SQL_METADATA_VER2_EXISTED = hBaseAdminTemplate.tableExists(HBaseTables.SQL_METADATA_VER2);
+
+        if (SQL_METADATA_VER2_EXISTED == false) {
+            logger.warn("Please create 'SqlMetaData_Ver2' table.");
+        }
+
+        SQL_METADATA_EXISTED = hBaseAdminTemplate.tableExists(HBaseTables.SQL_METADATA);
+
+        if (SQL_METADATA_EXISTED == true) {
+            logger.warn("SqlMetaData table exists. Recommend that only use SqlMetaData_Ver2 table.");
+        }
+
+        if(SQL_METADATA_EXISTED == false && SQL_METADATA_VER2_EXISTED == false) {
+            throw new RuntimeException("Please check for sqlMetaData_ver2 table in HBase. You Should create 'SqlMetaData_Ver2' table.");
+        }
+    }
+
+    @Override
+    public void insert(TSqlMetaData sqlMetaData) {
+        if (SQL_METADATA_VER2_EXISTED) {
+            hbaseSqlMetaDataDao.insert(sqlMetaData);
+            return;
+        } 
+
+        if (SQL_METADATA_EXISTED) {
+            hbaseSqlMetaDataPastVersionDao.insert(sqlMetaData);
+        }
+    }
+
+    public void setHbaseSqlMetaDataPastVersionDao(SqlMetaDataDao hbaseSqlMetaDataPastVersionDao) {
+        this.hbaseSqlMetaDataPastVersionDao = hbaseSqlMetaDataPastVersionDao;
+    }
+
+    public void setHbaseSqlMetaDataDao(SqlMetaDataDao hbaseSqlMetaDataDao) {
+        this.hbaseSqlMetaDataDao = hbaseSqlMetaDataDao;
+    }
+}

collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseSqlMetaDataDao.java

@@ -34,7 +34,7 @@
 /**
  * @author emeroad
  */
-@Repository
+//@Repository
 public class HbaseSqlMetaDataDao implements SqlMetaDataDao {
 
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
@@ -43,7 +43,7 @@ public class HbaseSqlMetaDataDao implements SqlMetaDataDao {
     private HbaseOperations2 hbaseTemplate;
 
     @Autowired
-    @Qualifier("metadataRowKeyDistributor")
+    @Qualifier("metadataRowKeyDistributor2")
     private RowKeyDistributorByHashPrefix rowKeyDistributorByHashPrefix;
 
     @Override
@@ -64,9 +64,9 @@ public void insert(TSqlMetaData sqlMetaData) {
         byte[] sqlBytes = Bytes.toBytes(sql);
 
         // added sqlBytes into qualifier intentionally not to conflict hashcode
-        put.addColumn(HBaseTables.SQL_METADATA_CF_SQL, sqlBytes, null);
+        put.addColumn(HBaseTables.SQL_METADATA_VER2_CF_SQL, sqlBytes, null);
 
-        hbaseTemplate.put(HBaseTables.SQL_METADATA, put);
+        hbaseTemplate.put(HBaseTables.SQL_METADATA_VER2, put);
     }
 
     private byte[] getDistributedKey(byte[] rowKey) {

collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseSqlMetaDataPastVersionDao.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2014 NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.navercorp.pinpoint.collector.dao.hbase;
+
+import com.navercorp.pinpoint.collector.dao.SqlMetaDataDao;
+import com.navercorp.pinpoint.common.bo.SqlMetaDataBo;
+import com.navercorp.pinpoint.common.hbase.HBaseTables;
+import com.navercorp.pinpoint.common.hbase.HbaseOperations2;
+import com.navercorp.pinpoint.thrift.dto.TSqlMetaData;
+import com.sematext.hbase.wd.RowKeyDistributorByHashPrefix;
+
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Repository;
+
+/**
+ * @author emeroad
+ */
+//@Repository
+public class HbaseSqlMetaDataPastVersionDao implements SqlMetaDataDao {
+
+    private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+    @Autowired
+    private HbaseOperations2 hbaseTemplate;
+
+    @Autowired
+    @Qualifier("metadataRowKeyDistributor")
+    private RowKeyDistributorByHashPrefix rowKeyDistributorByHashPrefix;
+
+    @Override
+    public void insert(TSqlMetaData sqlMetaData) {
+        if (sqlMetaData == null) {
+            throw new NullPointerException("sqlMetaData must not be null");
+        }
+        if (logger.isDebugEnabled()) {
+            logger.debug("insert:{}", sqlMetaData);
+        }
+
+        SqlMetaDataBo sqlMetaDataBo = new SqlMetaDataBo(sqlMetaData.getAgentId(), sqlMetaData.getAgentStartTime(), sqlMetaData.getSqlId());
+        final byte[] rowKey = getDistributedKey(sqlMetaDataBo.toRowKey());
+
+
+        Put put = new Put(rowKey);
+        String sql = sqlMetaData.getSql();
+        byte[] sqlBytes = Bytes.toBytes(sql);
+
+        // added sqlBytes into qualifier intentionally not to conflict hashcode
+        put.addColumn(HBaseTables.SQL_METADATA_CF_SQL, sqlBytes, null);
+
+        hbaseTemplate.put(HBaseTables.SQL_METADATA, put);
+    }
+
+    private byte[] getDistributedKey(byte[] rowKey) {
+        return rowKeyDistributorByHashPrefix.getDistributedKey(rowKey);
+    }
+}

collector/src/main/java/com/navercorp/pinpoint/collector/handler/SqlMetaDataHandler.java

@@ -28,12 +28,13 @@
 
 /**
  * @author emeroad
+ * @author minwoo.jung
  */
-@Service
+//@Service
 public class SqlMetaDataHandler implements RequestResponseHandler {
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
-    @Autowired
+//    @Autowired
     private SqlMetaDataDao sqlMetaDataDao;
 
     @Override
@@ -60,4 +61,8 @@ public class SqlMetaDataHandler implements RequestResponseHandler {
         }
         return new TResult(true);
     }
+
+    public void setSqlMetaDataDao(SqlMetaDataDao sqlMetaDataDao) {
+        this.sqlMetaDataDao = sqlMetaDataDao;
+    }
 }

collector/src/main/resources/applicationContext-collector.xml

@@ -128,4 +128,16 @@
     <bean id="timeSlot" class="com.navercorp.pinpoint.common.util.DefaultTimeSlot">
     </bean>
 
+    <bean id="sqlMetaDataHandler" class="com.navercorp.pinpoint.collector.handler.SqlMetaDataHandler">
+        <property name="sqlMetaDataDao" ref="hbaseSqlMetaDataCompatibility"/>
+    </bean>
+
+    <bean id="hbaseSqlMetaDataCompatibility" class="com.navercorp.pinpoint.collector.dao.hbase.HbaseSqlMetaDataCompatibility">
+        <property name="hbaseSqlMetaDataDao" ref="hbaseSqlMetaDataDao"/>
+        <property name="hbaseSqlMetaDataPastVersionDao" ref="hbaseSqlMetaDataPastVersionDao"/>
+    </bean>
+
+    <bean id="hbaseSqlMetaDataPastVersionDao" class="com.navercorp.pinpoint.collector.dao.hbase.HbaseSqlMetaDataPastVersionDao"/>
+    <bean id="hbaseSqlMetaDataDao" class="com.navercorp.pinpoint.collector.dao.hbase.HbaseSqlMetaDataDao"/>
+
 </beans>

collector/src/main/resources/applicationContext-hbase.xml

@@ -47,6 +47,10 @@
         <property name="configuration" ref="hbaseConfiguration"/>
         <property name="tableFactory" ref="connectionFactory"/>
     </bean>
+
+    <bean id="hBaseAdminTemplate" class="com.navercorp.pinpoint.common.hbase.HBaseAdminTemplate" destroy-method="close">
+        <constructor-arg ref="hbaseConfiguration" index="0"></constructor-arg>
+    </bean>
 
     <bean id="applicationTraceIndexDistributor" class="com.sematext.hbase.wd.RowKeyDistributorByHashPrefix">
         <constructor-arg ref="applicationTraceIndex"/>
@@ -87,6 +91,16 @@
         <constructor-arg type="int" value="32"/>
         <constructor-arg type="int" value="8"/>
     </bean>
+
+    <bean id="metadataRowKeyDistributor2" class="com.sematext.hbase.wd.RowKeyDistributorByHashPrefix">
+        <constructor-arg ref="metadataRangeHasher2"/>
+    </bean>
+
+    <bean id="metadataRangeHasher2" class="com.navercorp.pinpoint.common.hbase.distributor.RangeOneByteSimpleHash">
+        <constructor-arg type="int" value="0"/>
+        <constructor-arg type="int" value="36"/>
+        <constructor-arg type="int" value="32"/>
+    </bean>
 
     <bean id="acceptApplicationRowKeyDistributor" class="com.sematext.hbase.wd.RowKeyDistributorByHashPrefix">
         <constructor-arg ref="acceptApplicationHasher"/>

commons-hbase/src/main/java/com/navercorp/pinpoint/common/hbase/HBaseTables.java

@@ -60,6 +60,9 @@ public final class HBaseTables {
 
     public static final String SQL_METADATA = "SqlMetaData";
     public static final byte[] SQL_METADATA_CF_SQL = Bytes.toBytes("Sql");
+
+    public static final String SQL_METADATA_VER2 = "SqlMetaData_Ver2";
+    public static final byte[] SQL_METADATA_VER2_CF_SQL = Bytes.toBytes("Sql");
 
     public static final String STRING_METADATA = "StringMetaData";
     public static final byte[] STRING_METADATA_CF_STR = Bytes.toBytes("Str");

quickstart/conf/hbase/drop-hbase.txt

@@ -1,43 +1,47 @@
-
-disable 'AgentInfo'
-disable 'AgentStat'
-disable 'ApplicationIndex'
-
-disable 'StringMetaData'
-disable 'SqlMetaData'
-disable 'ApiMetaData'
-
-disable 'ApplicationTraceIndex'
-disable 'Traces'
-
-disable 'ApplicationMapStatisticsCaller'
-disable 'ApplicationMapStatisticsCallee'
-disable 'ApplicationMapStatisticsSelf'
-
-disable 'ApplicationStatistics'
-disable 'HostApplicationMap'
-disable 'HostApplicationMap_Ver2'
-
-
-drop 'AgentInfo'
-drop 'AgentStat'
-drop 'ApplicationIndex'
-
-drop 'StringMetaData'
-drop 'SqlMetaData'
-drop 'ApiMetaData'
-
-drop 'ApplicationTraceIndex'
-drop 'Traces'
-
-drop 'ApplicationMapStatisticsCaller'
-drop 'ApplicationMapStatisticsCallee'
-drop 'ApplicationMapStatisticsSelf'
-
-drop 'ApplicationStatistics'
-drop 'HostApplicationMap'
-drop 'HostApplicationMap_Ver2'
-
-list
-
+
+disable 'AgentInfo'
+disable 'AgentStat'
+disable 'ApplicationIndex'
+
+disable 'StringMetaData'
+disable 'ApiMetaData'
+
+disable 'SqlMetaData'
+disable 'SqlMetaData_Ver2'
+
+disable 'ApplicationTraceIndex'
+disable 'Traces'
+
+disable 'ApplicationMapStatisticsCaller'
+disable 'ApplicationMapStatisticsCallee'
+disable 'ApplicationMapStatisticsSelf'
+
+disable 'ApplicationStatistics'
+disable 'HostApplicationMap'
+disable 'HostApplicationMap_Ver2'
+
+
+drop 'AgentInfo'
+drop 'AgentStat'
+drop 'ApplicationIndex'
+
+drop 'StringMetaData'
+drop 'ApiMetaData'
+
+drop 'SqlMetaData'
+drop 'SqlMetaData_Ver2'
+
+drop 'ApplicationTraceIndex'
+drop 'Traces'
+
+drop 'ApplicationMapStatisticsCaller'
+drop 'ApplicationMapStatisticsCallee'
+drop 'ApplicationMapStatisticsSelf'
+
+drop 'ApplicationStatistics'
+drop 'HostApplicationMap'
+drop 'HostApplicationMap_Ver2'
+
+list
+
 exit