Skip to content

Commit

Permalink
Pre-compile pack/unpack function calls
Browse files Browse the repository at this point in the history
I noticed that pack/unpack functions from
https://github.com/dpkp/kafka-python/blob/master/kafka/protocol/types.py
might be slightly improved. I made pre-compilation for them. It gives
about 10% better performance compared to the current implementation.

Consumption of 100msg:
```
239884    0.187    0.000    0.287    0.000 types.py:18(_unpack) # new version
239884    0.192    0.000    0.323    0.000 types.py:17(_unpack)
```

I also made some profiling for producers/consumers. It gives about
1-1.5% time savings.
  • Loading branch information
billyevans authored and jeffwidman committed Oct 29, 2018
1 parent 4d13713 commit 3689da3
Showing 1 changed file with 29 additions and 13 deletions.
42 changes: 29 additions & 13 deletions kafka/protocol/types.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
from __future__ import absolute_import

from struct import pack, unpack, error
import struct
from struct import error

from kafka.protocol.abstract import AbstractType


def _pack(f, value):
try:
return pack(f, value)
return f(value)
except error as e:
raise ValueError("Error encountered when attempting to convert value: "
"{!r} to struct format: '{}', hit error: {}"
Expand All @@ -16,7 +17,7 @@ def _pack(f, value):

def _unpack(f, data):
try:
(value,) = unpack(f, data)
(value,) = f(data)
return value
except error as e:
raise ValueError("Error encountered when attempting to convert value: "
Expand All @@ -25,43 +26,55 @@ def _unpack(f, data):


class Int8(AbstractType):
_pack = struct.Struct('>b').pack
_unpack = struct.Struct('>b').unpack

@classmethod
def encode(cls, value):
return _pack('>b', value)
return _pack(cls._pack, value)

@classmethod
def decode(cls, data):
return _unpack('>b', data.read(1))
return _unpack(cls._unpack, data.read(1))


class Int16(AbstractType):
_pack = struct.Struct('>h').pack
_unpack = struct.Struct('>h').unpack

@classmethod
def encode(cls, value):
return _pack('>h', value)
return _pack(cls._pack, value)

@classmethod
def decode(cls, data):
return _unpack('>h', data.read(2))
return _unpack(cls._unpack, data.read(2))


class Int32(AbstractType):
_pack = struct.Struct('>i').pack
_unpack = struct.Struct('>i').unpack

@classmethod
def encode(cls, value):
return _pack('>i', value)
return _pack(cls._pack, value)

@classmethod
def decode(cls, data):
return _unpack('>i', data.read(4))
return _unpack(cls._unpack, data.read(4))


class Int64(AbstractType):
_pack = struct.Struct('>q').pack
_unpack = struct.Struct('>q').unpack

@classmethod
def encode(cls, value):
return _pack('>q', value)
return _pack(cls._pack, value)

@classmethod
def decode(cls, data):
return _unpack('>q', data.read(8))
return _unpack(cls._unpack, data.read(8))


class String(AbstractType):
Expand Down Expand Up @@ -108,13 +121,16 @@ def repr(cls, value):


class Boolean(AbstractType):
_pack = struct.Struct('>?').pack
_unpack = struct.Struct('>?').unpack

@classmethod
def encode(cls, value):
return _pack('>?', value)
return _pack(cls._pack, value)

@classmethod
def decode(cls, data):
return _unpack('>?', data.read(1))
return _unpack(cls._unpack, data.read(1))


class Schema(AbstractType):
Expand Down

0 comments on commit 3689da3

Please # to comment.