[pinpoint-apm#562] Improve logic inserting SqlMetaData data.

Create class to support to compatibility.
minwoo-jung · Jun 9, 2015 · fc6a68c · fc6a68c
1 parent 8269a8f
commit fc6a68c
Show file tree

Hide file tree

Showing 7 changed files with 190 additions and 7 deletions.
diff --git a/...c/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseSqlMetaDataCompatibility.java b/...c/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseSqlMetaDataCompatibility.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2014 NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.navercorp.pinpoint.collector.dao.hbase;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Repository;
+
+import com.navercorp.pinpoint.collector.dao.SqlMetaDataDao;
+import com.navercorp.pinpoint.common.hbase.HBaseAdminTemplate;
+import com.navercorp.pinpoint.common.hbase.HBaseTables;
+import com.navercorp.pinpoint.thrift.dto.TSqlMetaData;
+
+/**
+ * @author minwoo.jung
+ */
+//@Repository
+public class HbaseSqlMetaDataCompatibility implements SqlMetaDataDao {
+
+    private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+    private final boolean SQL_METADATA_VER2_EXISTED;
+
+    @Autowired
+    private SqlMetaDataDao hbaseSqlMetaDataPastVersionDao;
+
+    @Autowired
+    private SqlMetaDataDao hbaseSqlMetaDataDao;
+
+    @Autowired
+    public HbaseSqlMetaDataCompatibility(HBaseAdminTemplate hBaseAdminTemplate) {
+        SQL_METADATA_VER2_EXISTED = hBaseAdminTemplate.tableExists(HBaseTables.SQL_METADATA_VER2);
+
+        if (SQL_METADATA_VER2_EXISTED == false) {
+            logger.warn("Please create 'SqlMetaData_Ver2' table.");
+        }
+
+        if(hBaseAdminTemplate.tableExists(HBaseTables.SQL_METADATA) == false && SQL_METADATA_VER2_EXISTED == false) {
+            throw new RuntimeException("Please check for sqlMetaData_ver2 table in HBase. Need to create 'SqlMetaData_Ver2' table.");
+        }
+    }
+
+    @Override
+    public void insert(TSqlMetaData sqlMetaData) {
+        if (SQL_METADATA_VER2_EXISTED) {
+            hbaseSqlMetaDataDao.insert(sqlMetaData);
+        } else {
+            hbaseSqlMetaDataPastVersionDao.insert(sqlMetaData);
+        }
+    }
+
+    public void setHbaseSqlMetaDataPastVersionDao(SqlMetaDataDao hbaseSqlMetaDataPastVersionDao) {
+        this.hbaseSqlMetaDataPastVersionDao = hbaseSqlMetaDataPastVersionDao;
+    }
+
+    public void setHbaseSqlMetaDataDao(SqlMetaDataDao hbaseSqlMetaDataDao) {
+        this.hbaseSqlMetaDataDao = hbaseSqlMetaDataDao;
+    }
+}
diff --git a/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseSqlMetaDataDao.java b/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseSqlMetaDataDao.java
@@ -32,9 +32,9 @@
 import org.springframework.stereotype.Repository;
 
 /**
- * @author emeroad
+ * @author 
  */
-@Repository
+//@Repository
 public class HbaseSqlMetaDataDao implements SqlMetaDataDao {
 
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
@@ -43,7 +43,7 @@ public class HbaseSqlMetaDataDao implements SqlMetaDataDao {
     private HbaseOperations2 hbaseTemplate;
 
     @Autowired
-    @Qualifier("metadataRowKeyDistributor")
+    @Qualifier("metadataRowKeyDistributor2")
     private RowKeyDistributorByHashPrefix rowKeyDistributorByHashPrefix;
 
     @Override
@@ -64,9 +64,9 @@ public void insert(TSqlMetaData sqlMetaData) {
         byte[] sqlBytes = Bytes.toBytes(sql);
 
         // added sqlBytes into qualifier intentionally not to conflict hashcode
-        put.addColumn(HBaseTables.SQL_METADATA_CF_SQL, sqlBytes, null);
+        put.addColumn(HBaseTables.SQL_METADATA_VER2_CF_SQL, sqlBytes, null);
 
-        hbaseTemplate.put(HBaseTables.SQL_METADATA, put);
+        hbaseTemplate.put(HBaseTables.SQL_METADATA_VER2, put);
     }
 
     private byte[] getDistributedKey(byte[] rowKey) {

diff --git a/.../main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseSqlMetaDataPastVersionDao.java b/.../main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseSqlMetaDataPastVersionDao.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2014 NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.navercorp.pinpoint.collector.dao.hbase;
+
+import com.navercorp.pinpoint.collector.dao.SqlMetaDataDao;
+import com.navercorp.pinpoint.common.bo.SqlMetaDataBo;
+import com.navercorp.pinpoint.common.hbase.HBaseTables;
+import com.navercorp.pinpoint.common.hbase.HbaseOperations2;
+import com.navercorp.pinpoint.thrift.dto.TSqlMetaData;
+import com.sematext.hbase.wd.RowKeyDistributorByHashPrefix;
+
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Repository;
+
+/**
+ * @author emeroad
+ */
+//@Repository
+public class HbaseSqlMetaDataPastVersionDao implements SqlMetaDataDao {
+
+    private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+    @Autowired
+    private HbaseOperations2 hbaseTemplate;
+
+    @Autowired
+    @Qualifier("metadataRowKeyDistributor")
+    private RowKeyDistributorByHashPrefix rowKeyDistributorByHashPrefix;
+
+    @Override
+    public void insert(TSqlMetaData sqlMetaData) {
+        if (sqlMetaData == null) {
+            throw new NullPointerException("sqlMetaData must not be null");
+        }
+        if (logger.isDebugEnabled()) {
+            logger.debug("insert:{}", sqlMetaData);
+        }
+
+        SqlMetaDataBo sqlMetaDataBo = new SqlMetaDataBo(sqlMetaData.getAgentId(), sqlMetaData.getAgentStartTime(), sqlMetaData.getSqlId());
+        final byte[] rowKey = getDistributedKey(sqlMetaDataBo.toRowKey());
+
+
+        Put put = new Put(rowKey);
+        String sql = sqlMetaData.getSql();
+        byte[] sqlBytes = Bytes.toBytes(sql);
+
+        // added sqlBytes into qualifier intentionally not to conflict hashcode
+        put.addColumn(HBaseTables.SQL_METADATA_CF_SQL, sqlBytes, null);
+
+        hbaseTemplate.put(HBaseTables.SQL_METADATA, put);
+    }
+
+    private byte[] getDistributedKey(byte[] rowKey) {
+        return rowKeyDistributorByHashPrefix.getDistributedKey(rowKey);
+    }
+}
diff --git a/collector/src/main/java/com/navercorp/pinpoint/collector/handler/SqlMetaDataHandler.java b/collector/src/main/java/com/navercorp/pinpoint/collector/handler/SqlMetaDataHandler.java
@@ -24,16 +24,17 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.stereotype.Service;
 
 /**
  * @author emeroad
  */
-@Service
+//@Service
 public class SqlMetaDataHandler implements RequestResponseHandler {
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
-    @Autowired
+//    @Autowired
     private SqlMetaDataDao sqlMetaDataDao;
 
     @Override
@@ -60,4 +61,8 @@ public class SqlMetaDataHandler implements RequestResponseHandler {
         }
         return new TResult(true);
     }
+
+    public void setSqlMetaDataDao(SqlMetaDataDao sqlMetaDataDao) {
+        this.sqlMetaDataDao = sqlMetaDataDao;
+    }
 }
diff --git a/collector/src/main/resources/applicationContext-collector.xml b/collector/src/main/resources/applicationContext-collector.xml
@@ -134,4 +134,15 @@
         <constructor-arg ref="typeLoaderService"/>
     </bean>
 
+    <bean id="sqlMetaDataHandler" class="com.navercorp.pinpoint.collector.handler.SqlMetaDataHandler">
+        <property name="sqlMetaDataDao" ref="hbaseSqlMetaDataCompatibility"/>
+    </bean>
+
+    <bean id="hbaseSqlMetaDataCompatibility" class="com.navercorp.pinpoint.collector.dao.hbase.HbaseSqlMetaDataCompatibility">
+        <property name="hbaseSqlMetaDataDao" ref="hbaseSqlMetaDataDao"/>
+        <property name="hbaseSqlMetaDataPastVersionDao" ref="hbaseSqlMetaDataPastVersionDao"/>
+    </bean>
+
+    <bean id="hbaseSqlMetaDataPastVersionDao" class="com.navercorp.pinpoint.collector.dao.hbase.HbaseSqlMetaDataPastVersionDao"/>
+    <bean id="hbaseSqlMetaDataDao" class="com.navercorp.pinpoint.collector.dao.hbase.HbaseSqlMetaDataDao"/>
 </beans>
diff --git a/collector/src/main/resources/applicationContext-hbase.xml b/collector/src/main/resources/applicationContext-hbase.xml
@@ -47,6 +47,10 @@
         <property name="configuration" ref="hbaseConfiguration"/>
         <property name="tableFactory" ref="connectionFactory"/>
     </bean>
+
+    <bean id="hBaseAdminTemplate" class="com.navercorp.pinpoint.common.hbase.HBaseAdminTemplate" destroy-method="close">
+        <constructor-arg ref="hbaseConfiguration" index="0"></constructor-arg>
+    </bean>
 
     <bean id="applicationTraceIndexDistributor" class="com.sematext.hbase.wd.RowKeyDistributorByHashPrefix">
         <constructor-arg ref="applicationTraceIndex"/>
@@ -87,6 +91,16 @@
         <constructor-arg type="int" value="32"/>
         <constructor-arg type="int" value="8"/>
     </bean>
+
+    <bean id="metadataRowKeyDistributor2" class="com.sematext.hbase.wd.RowKeyDistributorByHashPrefix">
+        <constructor-arg ref="metadataRangeHasher2"/>
+    </bean>
+
+    <bean id="metadataRangeHasher2" class="com.navercorp.pinpoint.common.hbase.distributor.RangeOneByteSimpleHash">
+        <constructor-arg type="int" value="0"/>
+        <constructor-arg type="int" value="36"/>
+        <constructor-arg type="int" value="32"/>
+    </bean>
 
     <bean id="acceptApplicationRowKeyDistributor" class="com.sematext.hbase.wd.RowKeyDistributorByHashPrefix">
         <constructor-arg ref="acceptApplicationHasher"/>

diff --git a/commons/src/main/java/com/navercorp/pinpoint/common/hbase/HBaseTables.java b/commons/src/main/java/com/navercorp/pinpoint/common/hbase/HBaseTables.java
@@ -60,6 +60,9 @@ public final class HBaseTables {
 
     public static final String SQL_METADATA = "SqlMetaData";
     public static final byte[] SQL_METADATA_CF_SQL = Bytes.toBytes("Sql");
+
+    public static final String SQL_METADATA_VER2 = "SqlMetaData_Ver2";
+    public static final byte[] SQL_METADATA_VER2_CF_SQL = Bytes.toBytes("Sql");
 
     public static final String STRING_METADATA = "StringMetaData";
     public static final byte[] STRING_METADATA_CF_STR = Bytes.toBytes("Str");