EZP-30554: Purge several keys at once to simplify debug & reduce roun…

…d trips (#85)

* EZP-30554: Purge several keys at once to simplify debug & reduce round trips

* Remove http_cache.varnish_bulk_purge

* Add doc on requriment for varnsih-modules 0.10.2+

* Apply suggestions from code review

Co-Authored-By: Kamil Madejski <kmadejski@live.com>

docs/using_tags.md

@@ -9,7 +9,7 @@ They work in a similar way as [persistence cache tags in eZ Platform v2](https:/
 
 It works across all supported proxies _(see ["drivers"](drivers.md))_ by eZ Platform:
 - Symfony Proxy _(PHP based for single server usage, primarily for smaller web sites)_
-- [Varnish](https://varnish-cache.org/) with [xkey module](https://github.com/varnish/varnish-modules) _or_ [Varnish Plus](https://www.varnish-software.com/products/varnish-plus/) _(High performance reverse proxy)_
+- [Varnish](https://varnish-cache.org/) with [xkey module (0.10.2+)](https://github.com/varnish/varnish-modules) _or_ [Varnish Plus](https://www.varnish-software.com/products/varnish-plus/) _(High performance reverse proxy)_
 - [Fastly](https://www.fastly.com/) _(High performance reverse proxy, originally based on Varnish, worldwide as a CDN, driver available in eZ Platform Enterprise)_
 
 _In order to support several repositories on one installation, tags will be prefixed by

docs/varnish/README.md

@@ -5,6 +5,7 @@ Prerequisites
 -------------
 * A working Varnish 4.1 and higher setup with xkey module installed
   * _As of eZ Platform 2.5LTS the requirement is Varnish 5.1 (6.0LTS recommended and what we mainly test against)_
+  * ezplatform-http-cache 0.9+ requires varnish-modules 0.10.2 or higher, use 0.8 if you need to use varnish-modules 0.9
 * Varnish Plus comes with xkey out of the box and can also be used.
 
 Recommended VCL base files
@@ -23,7 +24,7 @@ For tuning the VCL further to you needs, see the following relevant examples:
 
 Example installation on Debian/Ubuntu:
 --------------------------------------
-Starting with Debian 9 and Ubuntu 16.10 installation of `xkey` VMOD is greatly
+Starting with Debian 9 and Ubuntu 18.04 installation of `xkey` VMOD is greatly
 simplified as new [varnish-modules](https://github.com/varnish/varnish-modules) package now exists.
 
 Install:

docs/varnish/vcl/varnish4.vcl

@@ -1,6 +1,6 @@
 // Varnish VCL for:
 // - Varnish 4.1 or higher (4.1LTS or 6.0LTS recommended, and is what we mainly test against)
-//   - Varnish xkey vmod (via varnish-modules package, or via Varnish Plus)
+//   - Varnish xkey vmod (via varnish-modules package 0.10.2 or higher, or via Varnish Plus)
 // - eZ Platform 1.13 with ezplatform-http-cache bundle
 //
 // Make sure to at least adjust default parameters.yml, defaults there reflect our testing needs with docker.

docs/varnish/vcl/varnish5.vcl

@@ -1,6 +1,6 @@
 // Varnish VCL for:
 // - Varnish 5.1 or higher (6.0LTS recommended, and is what we mainly test against)
-//   - Varnish xkey vmod (via varnish-modules package, or via Varnish Plus)
+//   - Varnish xkey vmod (via varnish-modules package 0.10.2 or higher, or via Varnish Plus)
 // - eZ Platform 2.5LTS or higher with ezplatform-http-cache (this) bundle
 //
 // Make sure to at least adjust default parameters.yml, defaults there reflect our testing needs with docker.

src/PurgeClient/VarnishPurgeClient.php

@@ -16,6 +16,8 @@ class VarnishPurgeClient implements PurgeClientInterface
 {
     const INVALIDATE_TOKEN_PARAM = 'http_cache.varnish_invalidate_token';
     const INVALIDATE_TOKEN_PARAM_NAME = 'x-invalidate-token';
+    const DEFAULT_HEADER_LENGTH = 7500;
+    const XKEY_TAG_SEPERATOR = ' ';
 
     /**
      * @var \FOS\HttpCacheBundle\CacheManager
@@ -44,20 +46,19 @@ public function purge($tags)
             return;
         }
 
-        // As key only support one tag being invalidated at a time, we loop.
-        // These will be queued by FOS\HttpCache\ProxyClient\Varnish and handled on kernel.terminate.
-        foreach (array_unique((array)$tags) as $tag) {
-            if (is_numeric($tag)) {
-                $tag = 'location-' . $tag;
-            }
+        // For 5.4/1.x BC make sure to map any int to location id tag
+        $tags = array_unique(array_map(static function ($tag) {
+            return is_numeric($tag) ? 'location-' . $tag : $tag;
+        },
+            (array)$tags
+        ));
 
-            $headers = [
-                'key' => $tag,
-                'Host' => empty($_SERVER['SERVER_NAME']) ? parse_url($this->configResolver->getParameter('http_cache.purge_servers')[0], PHP_URL_HOST) : $_SERVER['SERVER_NAME'],
-            ];
-
-            $headers = $this->addPurgeAuthHeader($headers);
+        $headers = $this->getPurgeHeaders();
+        $chunkSize = $this->determineTagsPerHeader($tags);
 
+        // NB! This requries varnish-modules 0.10.2, if you need support for varnish-modules 0.9.x, use ezplatform-http-cache 0.8.x
+        foreach (array_chunk($tags, $chunkSize) as $tagchunk) {
+            $headers['key'] = implode(' ', $tagchunk);
             $this->cacheManager->invalidatePath(
                 '/',
                 $headers
@@ -67,12 +68,8 @@ public function purge($tags)
 
     public function purgeAll()
     {
-        $headers = [
-            'key' => 'ez-all',
-            'Host' => empty($_SERVER['SERVER_NAME']) ? parse_url($this->configResolver->getParameter('http_cache.purge_servers')[0], PHP_URL_HOST) : $_SERVER['SERVER_NAME'],
-        ];
-
-        $headers = $this->addPurgeAuthHeader($headers);
+        $headers = $this->getPurgeHeaders();
+        $headers['key'] = 'ez-all';
 
         $this->cacheManager->invalidatePath(
             '/',
@@ -81,13 +78,16 @@ public function purgeAll()
     }
 
     /**
-     * Adds an Authentication header for Purge.
+     * Adds an generic headers needed for purge (Host and Authentication).
      *
-     * @param array $headers
      * @return array
      */
-    private function addPurgeAuthHeader(array $headers)
+    private function getPurgeHeaders()
     {
+        $headers = [
+            'Host' => empty($_SERVER['SERVER_NAME']) ? parse_url($this->configResolver->getParameter('http_cache.purge_servers')[0], PHP_URL_HOST) : $_SERVER['SERVER_NAME'],
+        ];
+
         if ($this->configResolver->hasParameter(self::INVALIDATE_TOKEN_PARAM)
             && null !== ($token = $this->configResolver->getParameter(self::INVALIDATE_TOKEN_PARAM))
         ) {
@@ -96,4 +96,23 @@ private function addPurgeAuthHeader(array $headers)
 
         return $headers;
     }
+
+    /**
+     * Get amount of tags per header, adapted from FOSHttpCache 2.x.
+     *
+     * @param array $tags
+     *
+     * @return int
+     */
+    private function determineTagsPerHeader(array $tags)
+    {
+        if (mb_strlen(implode(self::XKEY_TAG_SEPERATOR, $tags)) < self::DEFAULT_HEADER_LENGTH) {
+            return count($tags);
+        }
+
+        // Estimate the amount of tags by dividing the max header length by the largest tag (minus the glue length)
+        $tagsize = max(array_map('mb_strlen', $tags));
+
+        return floor(self::DEFAULT_HEADER_LENGTH / ($tagsize + strlen(self::XKEY_TAG_SEPERATOR))) ?: 1;
+    }
 }

tests/PurgeClient/VarnishPurgeClientTest.php

@@ -116,30 +116,34 @@ public function testPurgeOneLocationIdWithAuthHeaderAndKey()
      */
     public function testPurge(array $locationIds)
     {
-        foreach ($locationIds as $key => $locationId) {
-            $this->configResolver
-                ->expects($this->at($key * 3))
-                ->method('getParameter')
-                ->with('http_cache.purge_servers')
-                ->willReturn(['https://varnishpurgehost']);
+        $this->configResolver
+            ->expects($this->at(0))
+            ->method('getParameter')
+            ->with('http_cache.purge_servers')
+            ->willReturn(['https://varnishpurgehost']);
 
-            $this->configResolver
-                ->expects($this->at($key * 3 + 1))
-                ->method('hasParameter')
-                ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM)
-                ->willReturn(true);
+        $this->configResolver
+            ->expects($this->at(1))
+            ->method('hasParameter')
+            ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM)
+            ->willReturn(true);
 
-            $this->configResolver
-                ->expects($this->at($key * 3 + 2))
-                ->method('getParameter')
-                ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM)
-                ->willReturn(null);
+        $this->configResolver
+            ->expects($this->at(2))
+            ->method('getParameter')
+            ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM)
+            ->willReturn(null);
+
+        $keys = array_map(static function ($id) {
+            return "location-$id";
+        },
+            $locationIds
+        );
 
-            $this->cacheManager
-                ->expects($this->at($key))
-                ->method('invalidatePath')
-                ->with('/', ['key' => "location-$locationId", 'Host' => 'varnishpurgehost']);
-        }
+        $this->cacheManager
+            ->expects($this->once())
+            ->method('invalidatePath')
+            ->with('/', ['key' => implode(' ', $keys), 'Host' => 'varnishpurgehost']);
 
         $this->purgeClient->purge($locationIds);
     }
@@ -152,30 +156,34 @@ public function testPurgeWithAuthHeaderAndKey(array $locationIds = [])
         $tokenName = VarnishPurgeClient::INVALIDATE_TOKEN_PARAM_NAME;
         $token = 'secret-token-key';
 
-        foreach ($locationIds as $key => $locationId) {
-            $this->configResolver
-                ->expects($this->at($key * 3))
-                ->method('getParameter')
-                ->with('http_cache.purge_servers')
-                ->willReturn(['https://varnishpurgehost']);
+        $this->configResolver
+            ->expects($this->at(0))
+            ->method('getParameter')
+            ->with('http_cache.purge_servers')
+            ->willReturn(['https://varnishpurgehost']);
 
-            $this->configResolver
-                ->expects($this->at($key * 3 + 1))
-                ->method('hasParameter')
-                ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM)
-                ->willReturn(true);
+        $this->configResolver
+            ->expects($this->at(1))
+            ->method('hasParameter')
+            ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM)
+            ->willReturn(true);
 
-            $this->configResolver
-                ->expects($this->at($key * 3 + 2))
-                ->method('getParameter')
-                ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM)
-                ->willReturn($token);
-
-            $this->cacheManager
-                ->expects($this->at($key))
-                ->method('invalidatePath')
-                ->with('/', ['key' => "location-$locationId", 'Host' => 'varnishpurgehost', $tokenName => $token]);
-        }
+        $this->configResolver
+            ->expects($this->at(2))
+            ->method('getParameter')
+            ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM)
+            ->willReturn($token);
+
+        $keys = array_map(static function ($id) {
+            return "location-$id";
+        },
+            $locationIds
+        );
+
+        $this->cacheManager
+            ->expects($this->once())
+            ->method('invalidatePath')
+            ->with('/', ['key' => implode(' ', $keys), 'Host' => 'varnishpurgehost', $tokenName => $token]);
 
         $this->purgeClient->purge($locationIds);
     }