Merge pull request #497 from wikimedia/counter

Uses a UNIX semaphore to limit concurrent ebook-convert calls

README.md

@@ -40,7 +40,7 @@ This section deals with a full installation of WS Export on the local developmen
    For example, in Debian-based Linux distributions:
 
    ```console
-   apt install php-sqlite3 php-zip php-curl
+   apt install php-sqlite3 php-zip php-curl php-sysvsem
    ```
 
    Then create a `.env.local` file:

config/services.yaml

@@ -70,5 +70,11 @@ services:
             $rateLimit: '%env(int:APP_RATE_LIMIT)%'
             $rateDuration: '%env(int:APP_RATE_DURATION)%'
 
+    App\Util\Semaphore\Semaphore:
+        class: 'App\Util\Semaphore\UnixSemaphore'
+        arguments:
+            $semaphoreKey: 123455435644
+            $capacity: 4
+
     # add more service definitions when explicit configuration is needed
     # please note that last definitions always *replace* previous ones

src/Generator/ConvertGenerator.php

@@ -5,6 +5,7 @@
 use App\Book;
 use App\Exception\WsExportException;
 use App\FileCache;
+use App\Util\Semaphore\Semaphore;
 use App\Util\Util;
 use InvalidArgumentException;
 use Symfony\Component\HttpFoundation\Response;
@@ -77,9 +78,7 @@ public static function getSupportedTypes() {
 		return array_keys( self::$CONFIG );
 	}
 
-	/**
-	 * @var string
-	 */
+	/** @var string */
 	private $format;
 
 	/** @var int Command timeout in seconds. */
@@ -91,10 +90,14 @@ public static function getSupportedTypes() {
 	/** @var EpubGenerator */
 	private $epubGenerator;
 
-	public function __construct( int $timeout, FileCache $fileCache, EpubGenerator $epubGenerator ) {
+	/** @var ?Semaphore */
+	private $semaphore;
+
+	public function __construct( int $timeout, FileCache $fileCache, EpubGenerator $epubGenerator, ?Semaphore $semaphore = null ) {
 		$this->timeout = $timeout;
 		$this->fileCache = $fileCache;
 		$this->epubGenerator = $epubGenerator;
+		$this->semaphore = $semaphore;
 	}
 
 	/**
@@ -146,6 +149,14 @@ public function create( Book $book ) {
 	}
 
 	private function convert( $epubFileName, $outputFileName ) {
+		$lock = null;
+		if ( $this->semaphore !== null ) {
+			$lock = $this->semaphore->tryLock();
+			if ( $lock === null ) {
+				// Overload
+				throw new WsExportException( 'book-conversion', [], Response::HTTP_INTERNAL_SERVER_ERROR );
+			}
+		}
 		try {
 			$command = array_merge(
 				[ 'ebook-convert', $epubFileName, $outputFileName ],
@@ -156,6 +167,10 @@ private function convert( $epubFileName, $outputFileName ) {
 			$process->mustRun();
 		} catch ( ProcessTimedOutException $e ) {
 			throw new WsExportException( 'book-conversion', [], Response::HTTP_INTERNAL_SERVER_ERROR );
+		} finally {
+			if ( $lock !== null ) {
+				$lock->release();
+			}
 		}
 	}
 }

src/Util/Semaphore/Semaphore.php

@@ -0,0 +1,13 @@
+<?php
+
+namespace App\Util\Semaphore;
+
+/**
+ * A semaphore aka a lock allowing multiple threads/processes to acquire it.
+ */
+interface Semaphore {
+	/**
+	 * Attempts to lock the semaphore, returns null if it can't be locked because the semaphore is full.
+	 */
+	public function tryLock(): ?SemaphoreHandle;
+}

src/Util/Semaphore/SemaphoreHandle.php

@@ -0,0 +1,11 @@
+<?php
+
+namespace App\Util\Semaphore;
+
+/**
+ * A currently acquired semaphore
+ */
+interface SemaphoreHandle {
+	/** Releases the semaphore handle: allows another process to lock it. */
+	public function release(): void;
+}

src/Util/Semaphore/UnixSemaphore.php

@@ -0,0 +1,45 @@
+<?php
+
+namespace App\Util\Semaphore;
+
+use Exception;
+
+/**
+ * A semaphore backed by POSIX file APIs.
+ * Does not work on Windows.
+ */
+class UnixSemaphore implements Semaphore {
+	/** @var int */
+	private $semaphoreKey;
+	/** @var int */
+	private $capacity;
+	/** @var resource|null the lazily initialized semaphore descriptor */
+	private $semaphore;
+
+	/**
+	 * @param int $semaphoreKey unique identifier for this semaphore. Should be shared by all the processes using it.
+	 * @param int $capacity how many processes can lock the same semaphore.
+	 */
+	public function __construct( int $semaphoreKey, int $capacity ) {
+		$this->semaphoreKey = $semaphoreKey;
+		$this->capacity = $capacity;
+		$this->semaphore = null;
+	}
+
+	public function tryLock(): ?SemaphoreHandle {
+		if ( $this->semaphore === null ) {
+			$semaphore = sem_get( $this->semaphoreKey, $this->capacity );
+			if ( $semaphore === false ) {
+				throw new Exception( "Failed to create semaphore key $this->semaphoreKey" );
+			}
+			$this->semaphore = $semaphore;
+		}
+
+		if ( !sem_acquire( $this->semaphore, true ) ) {
+			return null; // Semaphore already full
+		}
+
+		// We return the handle
+		return new UnixSemaphoreHandle( $this->semaphore );
+	}
+}

src/Util/Semaphore/UnixSemaphoreHandle.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Util\Semaphore;
+
+class UnixSemaphoreHandle implements SemaphoreHandle {
+	/** @var resource */
+	private $semaphore;
+	/** @var bool */
+	private $isReleased;
+
+	public function __construct( $semaphore ) {
+		$this->semaphore = $semaphore;
+		$this->isReleased = false;
+	}
+
+	public function release(): void {
+		if ( $this->isReleased ) {
+			return; // Already released
+		}
+		$this->isReleased = true;
+		sem_release( $this->semaphore );
+	}
+
+	public function  __destruct() {
+		$this->release();
+	}
+}

tests/Util/Semaphore/UnixSemaphoreTest.php

@@ -0,0 +1,23 @@
+<?php
+
+namespace App\Tests\Util\Semaphore;
+
+use App\Util\Semaphore\SemaphoreHandle;
+use App\Util\Semaphore\UnixSemaphore;
+use PHPUnit\Framework\TestCase;
+
+class UnixSemaphoreTest extends TestCase {
+
+	/**
+	 * @covers \App\Util\Semaphore\UnixSemaphore
+	 * @covers \App\Util\Semaphore\UnixSemaphoreHandle
+	 */
+	public function testUnixSemaphore() {
+		$semaphore = new UnixSemaphore( 1, 1 );
+		$handle = $semaphore->tryLock();
+		$this->assertInstanceOf( SemaphoreHandle::class, $handle );
+		$this->assertNull( $semaphore->tryLock() );
+		$handle->release();
+		$this->assertInstanceOf( SemaphoreHandle::class, $semaphore->tryLock() );
+	}
+}