Skip to content

Commit

Permalink
pinpoint-apm#1819 trace format v2
Browse files Browse the repository at this point in the history
  - add RowKeyEncoder & RowKeyDecoder
  - add testcase
  • Loading branch information
emeroad committed Aug 3, 2016
1 parent 25b1b7b commit 46b525d
Show file tree
Hide file tree
Showing 21 changed files with 513 additions and 109 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,12 @@

package com.navercorp.pinpoint.collector.dao.hbase;

import com.navercorp.pinpoint.collector.dao.TracesDao;
import com.navercorp.pinpoint.collector.dao.TraceDao;

import com.navercorp.pinpoint.common.server.bo.BasicSpan;
import com.navercorp.pinpoint.common.server.bo.SpanChunkBo;
import com.navercorp.pinpoint.common.server.bo.filter.SpanEventFilter;
import com.navercorp.pinpoint.common.server.bo.serializer.RowKeyEncoder;
import com.navercorp.pinpoint.common.server.bo.serializer.trace.v1.AnnotationSerializer;
import com.navercorp.pinpoint.common.server.bo.serializer.trace.v1.SpanEventEncodingContext;
import com.navercorp.pinpoint.common.server.bo.serializer.trace.v1.SpanEventSerializer;
Expand All @@ -30,8 +31,6 @@
import com.navercorp.pinpoint.common.server.bo.SpanEventBo;
import static com.navercorp.pinpoint.common.hbase.HBaseTables.*;
import com.navercorp.pinpoint.common.hbase.HbaseOperations2;
import com.navercorp.pinpoint.common.server.util.SpanUtils;
import com.sematext.hbase.wd.AbstractRowKeyDistributor;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.hbase.client.Put;
import org.slf4j.Logger;
Expand All @@ -46,7 +45,7 @@
* @author emeroad
*/
@Repository
public class HbaseTraceDao implements TracesDao {
public class HbaseTraceDao implements TraceDao {

private final Logger logger = LoggerFactory.getLogger(this.getClass());

Expand All @@ -69,8 +68,8 @@ public class HbaseTraceDao implements TracesDao {
private AnnotationSerializer annotationSerializer;

@Autowired
@Qualifier("traceDistributor")
private AbstractRowKeyDistributor rowKeyDistributor;
@Qualifier("traceRowKeyEncoderV1")
private RowKeyEncoder<BasicSpan> rowKeyEncoder;

@Override
public void insert(final SpanBo spanBo) {
Expand All @@ -81,7 +80,7 @@ public void insert(final SpanBo spanBo) {

long acceptedTime = spanBo.getCollectorAcceptTime();

final byte[] rowKey = getDistributeRowKey(SpanUtils.getTransactionId(spanBo));
final byte[] rowKey = rowKeyEncoder.encodeRowKey(spanBo);
final Put put = new Put(rowKey, acceptedTime);

this.spanSerializer.serialize(spanBo, put, null);
Expand All @@ -96,10 +95,6 @@ public void insert(final SpanBo spanBo) {
}
}

private byte[] getDistributeRowKey(byte[] transactionId) {
return rowKeyDistributor.getDistributedKey(transactionId);
}

private void addNestedSpanEvent(Put put, SpanBo span) {
final List<SpanEventBo> spanEventBoList = span.getSpanEventBoList();
if (CollectionUtils.isEmpty(spanEventBoList)) {
Expand All @@ -114,7 +109,7 @@ private void addNestedSpanEvent(Put put, SpanBo span) {

@Override
public void insertSpanChunk(SpanChunkBo spanChunkBo) {
final byte[] rowKey = getDistributeRowKey(SpanUtils.getTransactionId(spanChunkBo));
final byte[] rowKey = rowKeyEncoder.encodeRowKey(spanChunkBo);
final long acceptedTime = acceptedTimeService.getAcceptedTime();
final Put put = new Put(rowKey, acceptedTime);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@

import com.navercorp.pinpoint.collector.dao.TraceDao;
import com.navercorp.pinpoint.common.hbase.HbaseOperations2;
import com.navercorp.pinpoint.common.server.bo.BasicSpan;
import com.navercorp.pinpoint.common.server.bo.SpanBo;
import com.navercorp.pinpoint.common.server.bo.SpanChunkBo;
import com.navercorp.pinpoint.common.server.bo.SpanEventBo;
import com.navercorp.pinpoint.common.server.bo.serializer.RowKeyEncoder;
import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.SpanChunkSerializerV2;
import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.SpanSerializerV2;
import com.navercorp.pinpoint.common.server.util.SpanUtils;
import com.sematext.hbase.wd.AbstractRowKeyDistributor;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.hbase.client.Put;
import org.slf4j.Logger;
Expand Down Expand Up @@ -40,8 +40,8 @@ public class HbaseTraceDaoV2 implements TraceDao {
private SpanChunkSerializerV2 spanChunkSerializer;

@Autowired
@Qualifier("traceV2Distributor")
private AbstractRowKeyDistributor rowKeyDistributor;
@Qualifier("traceRowKeyEncoderV2")
private RowKeyEncoder<BasicSpan> rowKeyEncoder;


@Override
Expand All @@ -53,7 +53,7 @@ public void insert(final SpanBo spanBo) {

long acceptedTime = spanBo.getCollectorAcceptTime();

final byte[] rowKey = getDistributeRowKey(SpanUtils.getTransactionId(spanBo));
final byte[] rowKey = this.rowKeyEncoder.encodeRowKey(spanBo);
final Put put = new Put(rowKey, acceptedTime);

this.spanSerializer.serialize(spanBo, put, null);
Expand All @@ -68,17 +68,10 @@ public void insert(final SpanBo spanBo) {



private byte[] getDistributeRowKey(byte[] transactionId) {
byte[] distributedKey = rowKeyDistributor.getDistributedKey(transactionId);
return distributedKey;
}



@Override
public void insertSpanChunk(SpanChunkBo spanChunkBo) {

final byte[] rowKey = getDistributeRowKey(SpanUtils.getTransactionId(spanChunkBo));
final byte[] rowKey = this.rowKeyEncoder.encodeRowKey(spanChunkBo);

final long acceptedTime = spanChunkBo.getCollectorAcceptTime();
final Put put = new Put(rowKey, acceptedTime);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import com.navercorp.pinpoint.thrift.dto.TSpanEvent;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Required;
import org.springframework.stereotype.Component;

import java.util.ArrayList;
Expand All @@ -38,12 +39,12 @@ public class SpanFactory {
public SpanFactory() {
}

@Autowired
@Autowired(required = false)
public void setSpanEventFilter(SpanEventFilter spanEventFilter) {
this.spanEventFilter = spanEventFilter;
}

@Autowired
@Autowired(required = false)
public void setAcceptedTimeService(AcceptedTimeService acceptedTimeService) {
this.acceptedTimeService = acceptedTimeService;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package com.navercorp.pinpoint.common.server.bo.serializer;

/**
* @author Woonduk Kang(emeroad)
*/
public interface RowKeyDecoder<V> {

V decodeRowKey(byte[] rowkey);

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package com.navercorp.pinpoint.common.server.bo.serializer;

/**
* @author Woonduk Kang(emeroad)
*/
public interface RowKeyEncoder<V> {

byte[] encodeRowKey(V value);

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package com.navercorp.pinpoint.common.server.bo.serializer.trace.v1;

import com.navercorp.pinpoint.common.server.bo.serializer.RowKeyDecoder;
import com.navercorp.pinpoint.common.util.BytesUtils;
import com.navercorp.pinpoint.common.util.TransactionId;
import com.sematext.hbase.wd.AbstractRowKeyDistributor;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;

/**
* @author Woonduk Kang(emeroad)
*/
@Component
public class TraceRowKeyDecoderV1 implements RowKeyDecoder<TransactionId> {

public static final int AGENT_NAME_MAX_LEN = TraceRowKeyEncoderV1.AGENT_NAME_MAX_LEN;
public static final int DISTRIBUTE_HASH_SIZE = TraceRowKeyEncoderV1.DISTRIBUTE_HASH_SIZE;

private final int distributeHashSize;


public TraceRowKeyDecoderV1() {
this(DISTRIBUTE_HASH_SIZE);
}

public TraceRowKeyDecoderV1(int distributeHashSize) {
this.distributeHashSize = distributeHashSize;
}

@Override
public TransactionId decodeRowKey(byte[] rowkey) {
if (rowkey == null) {
throw new NullPointerException("rowkey must not be null");
}

return readTransactionId(rowkey, distributeHashSize);
}

private TransactionId readTransactionId(byte[] rowKey, int offset) {

String agentId = BytesUtils.toStringAndRightTrim(rowKey, offset, AGENT_NAME_MAX_LEN);
long agentStartTime = BytesUtils.bytesToLong(rowKey, offset + AGENT_NAME_MAX_LEN);
long transactionSequence = BytesUtils.bytesToLong(rowKey, offset + BytesUtils.LONG_BYTE_LENGTH + AGENT_NAME_MAX_LEN);

return new TransactionId(agentId, agentStartTime, transactionSequence);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package com.navercorp.pinpoint.common.server.bo.serializer.trace.v1;

import com.navercorp.pinpoint.common.PinpointConstants;
import com.navercorp.pinpoint.common.server.bo.BasicSpan;
import com.navercorp.pinpoint.common.server.bo.serializer.RowKeyEncoder;
import com.navercorp.pinpoint.common.util.BytesUtils;
import com.sematext.hbase.wd.AbstractRowKeyDistributor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;

/**
* @author Woonduk Kang(emeroad)
*/
@Component
public class TraceRowKeyEncoderV1 implements RowKeyEncoder<BasicSpan> {

public static final int AGENT_NAME_MAX_LEN = PinpointConstants.AGENT_NAME_MAX_LEN;
public static final int DISTRIBUTE_HASH_SIZE = 1;

private final AbstractRowKeyDistributor rowKeyDistributor;


@Autowired
public TraceRowKeyEncoderV1(@Qualifier("traceDistributor") AbstractRowKeyDistributor rowKeyDistributor) {
if (rowKeyDistributor == null) {
throw new NullPointerException("rowKeyDistributor must not be null");
}

this.rowKeyDistributor = rowKeyDistributor;
}

public byte[] encodeRowKey(BasicSpan basicSpan) {
if (basicSpan == null) {
throw new NullPointerException("basicSpan must not be null");
}

byte[] rowKey = BytesUtils.stringLongLongToBytes(basicSpan.getTraceAgentId(), AGENT_NAME_MAX_LEN, basicSpan.getTraceAgentStartTime(), basicSpan.getTraceTransactionSequence());
return wrapDistributedRowKey(rowKey);
}

private byte[] wrapDistributedRowKey(byte[] rowKey) {
return rowKeyDistributor.getDistributedKey(rowKey);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package com.navercorp.pinpoint.common.server.bo.serializer.trace.v1;

import com.navercorp.pinpoint.common.PinpointConstants;
import com.navercorp.pinpoint.common.server.bo.BasicSpan;
import com.navercorp.pinpoint.common.server.bo.serializer.RowKeyEncoder;
import com.navercorp.pinpoint.common.util.BytesUtils;
import com.navercorp.pinpoint.common.util.TransactionId;
import com.sematext.hbase.wd.AbstractRowKeyDistributor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;

/**
* @author Woonduk Kang(emeroad)
*/
@Component
public class TransactionIdRowKeyEncoderV1 implements RowKeyEncoder<TransactionId> {

public static final int AGENT_NAME_MAX_LEN = TraceRowKeyEncoderV1.AGENT_NAME_MAX_LEN;
public static final int DISTRIBUTE_HASH_SIZE = TraceRowKeyEncoderV1.DISTRIBUTE_HASH_SIZE;

private final AbstractRowKeyDistributor rowKeyDistributor;

@Autowired
public TransactionIdRowKeyEncoderV1(@Qualifier("traceDistributor") AbstractRowKeyDistributor rowKeyDistributor) {
if (rowKeyDistributor == null) {
throw new NullPointerException("rowKeyDistributor must not be null");
}

this.rowKeyDistributor = rowKeyDistributor;
}

public byte[] encodeRowKey(TransactionId transactionId) {
if (transactionId == null) {
throw new NullPointerException("transactionId must not be null");
}

byte[] rowKey = BytesUtils.stringLongLongToBytes(transactionId.getAgentId(), AGENT_NAME_MAX_LEN, transactionId.getAgentStartTime(), transactionId.getTransactionSequence());
return wrapDistributedRowKey(rowKey);
}

private byte[] wrapDistributedRowKey(byte[] rowKey) {
return rowKeyDistributor.getDistributedKey(rowKey);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2;

import com.navercorp.pinpoint.common.server.bo.serializer.RowKeyDecoder;
import com.navercorp.pinpoint.common.util.BytesUtils;
import com.navercorp.pinpoint.common.util.TransactionId;
import org.springframework.stereotype.Component;

/**
* @author Woonduk Kang(emeroad)
*/
@Component
public class TraceRowKeyDecoderV2 implements RowKeyDecoder<TransactionId> {

public static final int AGENT_NAME_MAX_LEN = TraceRowKeyEncoderV2.AGENT_NAME_MAX_LEN;
public static final int DISTRIBUTE_HASH_SIZE = TraceRowKeyEncoderV2.DISTRIBUTE_HASH_SIZE;

private final int distributeHashSize;


public TraceRowKeyDecoderV2() {
this(DISTRIBUTE_HASH_SIZE);
}

public TraceRowKeyDecoderV2(int distributeHashSize) {
this.distributeHashSize = distributeHashSize;
}


@Override
public TransactionId decodeRowKey(byte[] rowkey) {
if (rowkey == null) {
throw new NullPointerException("rowkey must not be null");
}

return readTransactionId(rowkey, distributeHashSize);
}

private TransactionId readTransactionId(byte[] rowKey, int offset) {

String agentId = BytesUtils.toStringAndRightTrim(rowKey, offset, AGENT_NAME_MAX_LEN);
long agentStartTime = BytesUtils.bytesToLong(rowKey, offset + AGENT_NAME_MAX_LEN);
long transactionSequence = BytesUtils.bytesToLong(rowKey, offset + BytesUtils.LONG_BYTE_LENGTH + AGENT_NAME_MAX_LEN);

return new TransactionId(agentId, agentStartTime, transactionSequence);
}

// for test
public TransactionId readTransactionId(byte[] rowKey) {
return readTransactionId(rowKey, 0);
}
}
Loading

0 comments on commit 46b525d

Please # to comment.