Add igbinary serializer support

nerou42 · Dec 5, 2023 · 8c54151 · 8c54151
1 parent e58eebb
commit 8c54151
Show file tree

Hide file tree

Showing 5 changed files with 129 additions and 29 deletions.
diff --git a/README.md b/README.md
@@ -23,6 +23,7 @@ Use composer to install this library:
 There are pretty much no dependencies with some exceptions:
 
 - If you want to use the `toJSONFile()` method, you need to install `ext-json` (PHP's PECL JSON extension) as well.
+- If you want to use the igbinary serializer, `ext-igbinary` is required. See [php-ext-igbinary](https://github.com/igbinary/igbinary).
 - If you want to use LZ4 compression, `ext-lz4` is required. See [php-ext-lz4](https://github.com/kjdev/php-ext-lz4).
 
 ## Usage
@@ -48,6 +49,8 @@ The constructor of `LargeArrayBuffer` provides some options:
 
 1. You can set the threshold when to move the data to disk. When pushing data to the buffer, it is stored in memory until it gets too large.
     E.g.: `new LargeArrayBuffer(512);` to set a 512 MiB threshold. 
+1. You can choose either the PHP serializer or the [igbinary](https://github.com/igbinary/igbinary) serializer (PHP serializer is default).
+    E.g.: `new LargeArrayBuffer(serializer: LargeArrayBuffer::COMPRESSION_IGBINARY);`
 1. You can enable GZIP or LZ4 compression for the serialized items. Although this is recommended only if your items are pretty big like > 1 KiB each. E.g.: `new LargeArrayBuffer(compression: LargeArrayBuffer::COMPRESSION_GZIP);`. Note, that LZ4 compression requires [ext-lz4](https://github.com/kjdev/php-ext-lz4) to be installed.
 
 ### Read from the buffer
@@ -80,16 +83,22 @@ To put it in one sentence: This library uses [php://temp](https://www.php.net/ma
 
 A benchmark with 1 million measurements (consisting of DateTimeImmutable, int and float) using PHP 8.1 with 10 iterations comparing a normal array with the LargeArrayBuffer gave the following results (LargeArrayBuffer was configured with a memory limit of 128 MiB):
 
-| Action | Consumed time | Consumed memory | Buffer size |
-| :--- | ---: | ---: | ---: |
-| Fill array | 1.42 s | 490 MiB | NA |
-| Iterate over array | 0.11 s | 492 MiB | NA |
-| Fill buffer | 3.8 s | 0 B | 378.7 MiB |
-| Iterate over buffer | 2.73 s | 0 B | 378.7 MiB |
-| Fill buffer (GZIP) | 33.22 s | 0 B | 192.5 MiB |
-| Iterate over buffer (GZIP) | 5.84 s | 0 B | 192.5 MiB |
-| Fill buffer (LZ4) | 3.8 s | 0 B | 241 MiB |
-| Iterate over buffer (LZ4) | 2.75 s | 0 B | 241 MiB |
+| Action | Serializer | Compression | Consumed time | Consumed memory | Buffer size |
+| :--- | :--- | :--- | ---: | ---: | ---: |
+| Fill array | none | none | 1.57 s | 490.0 MiB | NA |
+| Iterate over array | none | none | 0.27 s | 492.0 MiB | NA |
+| Fill buffer | PHP | none | 4.27 s | 0 B | 378.7 MiB |
+| Iterate over buffer | PHP | none | 2.85 s | 0 B | 378.7 MiB |
+| Fill buffer | PHP | GZIP | 24.76 s | 0 B | 192.5 MiB |
+| Iterate over buffer | PHP | GZIP | 6.79 s | 0 B | 192.5 MiB |
+| Fill buffer | PHP | LZ4 | 4.89 s | 0 B | 241.0 MiB |
+| Iterate over buffer | PHP | LZ4 | 2.93 s | 0 B | 241.0 MiB |
+| Fill buffer | igbinary | none | 4.26 s | 0 B | 319.1 MiB |
+| Iterate over buffer | igbinary | none | 3.41 s | 0 B | 319.1 MiB |
+| Fill buffer | igbinary | GZIP | 21.50 s | 0 B | 173.2 MiB |
+| Iterate over buffer | igbinary | GZIP | 4.80 s | 0 B | 173.2 MiB |
+| Fill buffer | igbinary | LZ4 | 4.38 s | 0 B | 195.1 MiB |
+| Iterate over buffer | igbinary | LZ4 | 3.17 s | 0 B | 195.1 MiB |
 
 Note: 
 

diff --git a/bench/benchmark.php b/bench/benchmark.php
@@ -59,7 +59,7 @@ function printResult(string $label, array $metrics, string $key, int $tabs = 1,
   // normal buffer
   $start = microtime(true);
   $memBefore = memory_get_usage(true);
-  $buf = new LargeArrayBuffer(256);
+  $buf = new LargeArrayBuffer(128);
   $bench->bufferMeasurementsFill($buf);
   $metrics['fill_buffer'][] = [
     'time' => microtime(true) - $start,
@@ -79,7 +79,7 @@ function printResult(string $label, array $metrics, string $key, int $tabs = 1,
   // buffer with GZIP
   $start = microtime(true);
   $memBefore = memory_get_usage(true);
-  $buf = new LargeArrayBuffer(256, compression: LargeArrayBuffer::COMPRESSION_GZIP);
+  $buf = new LargeArrayBuffer(128, compression: LargeArrayBuffer::COMPRESSION_GZIP);
   $bench->bufferMeasurementsFill($buf);
   $metrics['fill_buffer_gz'][] = [
     'time' => microtime(true) - $start,
@@ -100,7 +100,7 @@ function printResult(string $label, array $metrics, string $key, int $tabs = 1,
   if(function_exists('lz4_compress')){
     $start = microtime(true);
     $memBefore = memory_get_usage(true);
-    $buf = new LargeArrayBuffer(256, compression: LargeArrayBuffer::COMPRESSION_LZ4);
+    $buf = new LargeArrayBuffer(128, compression: LargeArrayBuffer::COMPRESSION_LZ4);
     $bench->bufferMeasurementsFill($buf);
     $metrics['fill_buffer_lz4'][] = [
       'time' => microtime(true) - $start,
@@ -118,16 +118,90 @@ function printResult(string $label, array $metrics, string $key, int $tabs = 1,
     unset($buf);
   }
 
+  if(function_exists('igbinary_serialize')){
+    // normal buffer with igbinary
+    $start = microtime(true);
+    $memBefore = memory_get_usage(true);
+    $buf = new LargeArrayBuffer(128, serializer: LargeArrayBuffer::SERIALIZER_IGBINARY);
+    $bench->bufferMeasurementsFill($buf);
+    $metrics['fill_buffer_ig'][] = [
+      'time' => microtime(true) - $start,
+      'mem' => memory_get_usage(true) - $memBefore,
+      'size' => $buf->getSize()
+    ];
+
+    $start = microtime(true);
+    $bench->bufferMeasurementsIterate($buf);
+    $metrics['iterate_buffer_ig'][] = [
+      'time' => microtime(true) - $start,
+      'mem' => memory_get_usage(true) - $memBefore,
+      'size' => $buf->getSize()
+    ];
+    unset($buf);
+
+    // buffer with GZIP and igbinary
+    $start = microtime(true);
+    $memBefore = memory_get_usage(true);
+    $buf = new LargeArrayBuffer(128, serializer: LargeArrayBuffer::SERIALIZER_IGBINARY, compression: LargeArrayBuffer::COMPRESSION_GZIP);
+    $bench->bufferMeasurementsFill($buf);
+    $metrics['fill_buffer_gz_ig'][] = [
+      'time' => microtime(true) - $start,
+      'mem' => memory_get_usage(true) - $memBefore,
+      'size' => $buf->getSize()
+    ];
+
+    $start = microtime(true);
+    $bench->bufferMeasurementsIterate($buf);
+    $metrics['iterate_buffer_gz_ig'][] = [
+      'time' => microtime(true) - $start,
+      'mem' => memory_get_usage(true) - $memBefore,
+      'size' => $buf->getSize()
+    ];
+    unset($buf);
+
+    // buffer with LZ4 and igbinary
+    if(function_exists('lz4_compress')){
+      $start = microtime(true);
+      $memBefore = memory_get_usage(true);
+      $buf = new LargeArrayBuffer(128, serializer: LargeArrayBuffer::SERIALIZER_IGBINARY, compression: LargeArrayBuffer::COMPRESSION_LZ4);
+      $bench->bufferMeasurementsFill($buf);
+      $metrics['fill_buffer_lz4_ig'][] = [
+        'time' => microtime(true) - $start,
+        'mem' => memory_get_usage(true) - $memBefore,
+        'size' => $buf->getSize()
+      ];
+
+      $start = microtime(true);
+      $bench->bufferMeasurementsIterate($buf);
+      $metrics['iterate_buffer_lz4_ig'][] = [
+        'time' => microtime(true) - $start,
+        'mem' => memory_get_usage(true) - $memBefore,
+        'size' => $buf->getSize()
+      ];
+      unset($buf);
+    }
+  }
+
   unset($bench);
 }
 
-printResult('Fill array', $metrics, 'fill_array', 3);
-printResult('Iterate over array', $metrics, 'iterate_array', 2);
-printResult('Fill buffer', $metrics, 'fill_buffer', 3, true);
-printResult('Iterate over buffer', $metrics, 'iterate_buffer', 2, true);
-printResult('Fill buffer (GZIP)', $metrics, 'fill_buffer_gz', 2, true);
-printResult('Iterate over buffer (GZIP)', $metrics, 'iterate_buffer_gz', 1, true);
+printResult('Fill array', $metrics, 'fill_array', 4);
+printResult('Iterate over array', $metrics, 'iterate_array', 3);
+printResult('Fill buffer', $metrics, 'fill_buffer', 4, true);
+printResult('Iterate over buffer', $metrics, 'iterate_buffer', 3, true);
+printResult('Fill buffer (GZIP)', $metrics, 'fill_buffer_gz', 3, true);
+printResult('Iterate over buffer (GZIP)', $metrics, 'iterate_buffer_gz', 2, true);
 if(function_exists('lz4_compress')){
-  printResult('Fill buffer (LZ4)', $metrics, 'fill_buffer_lz4', 2, true);
-  printResult('Iterate over buffer (LZ4)', $metrics, 'iterate_buffer_lz4', 1, true);
+  printResult('Fill buffer (LZ4)', $metrics, 'fill_buffer_lz4', 3, true);
+  printResult('Iterate over buffer (LZ4)', $metrics, 'iterate_buffer_lz4', 2, true);
+}
+if(function_exists('igbinary_serialize')){
+  printResult('Fill buffer (igbinary)', $metrics, 'fill_buffer_ig', 2, true);
+  printResult('Iterate over buffer (igbinary)', $metrics, 'iterate_buffer_ig', 1, true);
+  printResult('Fill buffer (GZIP, igbinary)', $metrics, 'fill_buffer_gz_ig', 2, true);
+  printResult('Iterate over buffer (GZIP, igbinary)', $metrics, 'iterate_buffer_gz_ig', 1, true);
+  if(function_exists('lz4_compress')){
+    printResult('Fill buffer (LZ4, igbinary)', $metrics, 'fill_buffer_lz4_ig', 2, true);
+    printResult('Iterate over buffer (LZ4, igbinary)', $metrics, 'iterate_buffer_lz4_ig', 1, true);
+  }
 }
diff --git a/composer.json b/composer.json
@@ -1,6 +1,6 @@
 {
     "name": "nerou/large-array-buffer",
-    "version": "0.2",
+    "version": "0.3",
     "type": "library",
     "license": "MIT",
     "authors": [
@@ -13,7 +13,8 @@
     ],
     "suggest": {
         "ext-json": "Requirement of toJSONFile() method",
-        "ext-lz4": "To enable support of LZ4 compression"
+        "ext-lz4": "To enable support of LZ4 compression",
+        "ext-igbinary": "To enable support for igbinary serializer"
     },
     "require": {
         "php": ">=8.0 <8.4"

diff --git a/src/LargeArrayBuffer.php b/src/LargeArrayBuffer.php
@@ -12,7 +12,7 @@
 class LargeArrayBuffer implements \Iterator, \Countable {
 
   public const SERIALIZER_PHP = 1;
-  //public const SERIALIZER_JSON = 2;
+  public const SERIALIZER_IGBINARY = 2;
 
   public const COMPRESSION_NONE = 0;
   public const COMPRESSION_GZIP = 1;
@@ -51,11 +51,16 @@ class LargeArrayBuffer implements \Iterator, \Countable {
    * @param int $maxMemoryMiB maximum memory usage in MiB, when more data is pushed, disk space is used
    * @psalm-param self::SERIALIZER_* $serializer
    * @psalm-param self::COMPRESSION_* $compression
-   * @throws \InvalidArgumentException if an unsupported compression or serialization was requested
+   * @throws \InvalidArgumentException if an unsupported serialization was requested
+   * @throws \InvalidArgumentException if an unsupported compression was requested
    * @throws \RuntimeException if php://temp could not be opened
    */
   public function __construct(int $maxMemoryMiB = 1024, int $serializer = self::SERIALIZER_PHP, int $compression = self::COMPRESSION_NONE) {
     $this->serializer = $serializer;
+    if($this->serializer === self::SERIALIZER_IGBINARY && !function_exists('igbinary_serialize')){
+      throw new \InvalidArgumentException('igbinary serializer was requested, but ext-igbinary is not installed');
+    }
+
     $this->compression = $compression;
     if($this->compression === self::COMPRESSION_LZ4 && !function_exists('lz4_compress')){
       throw new \InvalidArgumentException('LZ4 compression was requested, but ext-lz4 is not installed');
@@ -74,7 +79,7 @@ public function __construct(int $maxMemoryMiB = 1024, int $serializer = self::SE
    */
   public function push(mixed $item): void {
     $serialized = match($this->serializer){
-      //self::SERIALIZER_JSON => json_encode($item, JSON_THROW_ON_ERROR),
+      self::SERIALIZER_IGBINARY => igbinary_serialize($item),
       default => serialize($item)
     };
     /** @var string|false $compressed */
@@ -139,7 +144,7 @@ public function current(): mixed {
     }
     /** @psalm-var E $res */
     $res = match($this->serializer){
-      //self::SERIALIZER_JSON => json_decode($this->current, flags: JSON_THROW_ON_ERROR),
+      self::SERIALIZER_IGBINARY => igbinary_unserialize($this->current),
       default => unserialize($this->current)
     };
     return $res;

diff --git a/test/LargeArrayBufferTest.php b/test/LargeArrayBufferTest.php
@@ -35,8 +35,7 @@ public function provideObject(): array {
     $o = $this->getObject();
     return [
       [$o, LargeArrayBuffer::SERIALIZER_PHP, LargeArrayBuffer::COMPRESSION_NONE],
-      //[$o, LargeArrayBuffer::SERIALIZER_JSON, LargeArrayBuffer::COMPRESSION_NONE],
-      [$o, LargeArrayBuffer::SERIALIZER_PHP, LargeArrayBuffer::COMPRESSION_GZIP]
+      [$o, LargeArrayBuffer::SERIALIZER_PHP, LargeArrayBuffer::COMPRESSION_GZIP],
     ];
   }
 
@@ -51,6 +50,18 @@ public function testReadWrite(object $o, int $serializer, int $compression): voi
     $this->assertEquals($o, $buf->current());
   }
 
+  /**
+   * @requires extension igbinary
+   */
+  public function testReadWriteIgbinary(): void {
+    $o = $this->getObject();
+    $buf = new LargeArrayBuffer(serializer: LargeArrayBuffer::SERIALIZER_IGBINARY);
+    $buf->push($o);
+    $buf->rewind();
+    $buf->next();
+    $this->assertEquals($o, $buf->current());
+  }
+
   /**
    * @requires extension lz4
    */