From 7c55fedc4756743bbc190c56ac8dbcf048f67258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20R?= Date: Tue, 21 May 2019 10:15:07 +0200 Subject: [PATCH] EZP-30554: Purge several keys at once to simplify debug & reduce round trips (#85) * EZP-30554: Purge several keys at once to simplify debug & reduce round trips * Remove http_cache.varnish_bulk_purge * Add doc on requriment for varnsih-modules 0.10.2+ * Apply suggestions from code review Co-Authored-By: Kamil Madejski --- docs/using_tags.md | 2 +- docs/varnish/README.md | 3 +- docs/varnish/vcl/varnish4.vcl | 2 +- docs/varnish/vcl/varnish5.vcl | 2 +- src/PurgeClient/VarnishPurgeClient.php | 61 ++++++++----- tests/PurgeClient/VarnishPurgeClientTest.php | 94 +++++++++++--------- 6 files changed, 96 insertions(+), 68 deletions(-) diff --git a/docs/using_tags.md b/docs/using_tags.md index 3433edda..fcc01b71 100644 --- a/docs/using_tags.md +++ b/docs/using_tags.md @@ -9,7 +9,7 @@ They work in a similar way as [persistence cache tags in eZ Platform v2](https:/ It works across all supported proxies _(see ["drivers"](drivers.md))_ by eZ Platform: - Symfony Proxy _(PHP based for single server usage, primarily for smaller web sites)_ -- [Varnish](https://varnish-cache.org/) with [xkey module](https://github.com/varnish/varnish-modules) _or_ [Varnish Plus](https://www.varnish-software.com/products/varnish-plus/) _(High performance reverse proxy)_ +- [Varnish](https://varnish-cache.org/) with [xkey module (0.10.2+)](https://github.com/varnish/varnish-modules) _or_ [Varnish Plus](https://www.varnish-software.com/products/varnish-plus/) _(High performance reverse proxy)_ - [Fastly](https://www.fastly.com/) _(High performance reverse proxy, originally based on Varnish, worldwide as a CDN, driver available in eZ Platform Enterprise)_ _In order to support several repositories on one installation, tags will be prefixed by diff --git a/docs/varnish/README.md b/docs/varnish/README.md index 91479e61..0a441f2c 100644 --- a/docs/varnish/README.md +++ b/docs/varnish/README.md @@ -5,6 +5,7 @@ Prerequisites ------------- * A working Varnish 4.1 and higher setup with xkey module installed * _As of eZ Platform 2.5LTS the requirement is Varnish 5.1 (6.0LTS recommended and what we mainly test against)_ + * ezplatform-http-cache 0.9+ requires varnish-modules 0.10.2 or higher, use 0.8 if you need to use varnish-modules 0.9 * Varnish Plus comes with xkey out of the box and can also be used. Recommended VCL base files @@ -23,7 +24,7 @@ For tuning the VCL further to you needs, see the following relevant examples: Example installation on Debian/Ubuntu: -------------------------------------- -Starting with Debian 9 and Ubuntu 16.10 installation of `xkey` VMOD is greatly +Starting with Debian 9 and Ubuntu 18.04 installation of `xkey` VMOD is greatly simplified as new [varnish-modules](https://github.com/varnish/varnish-modules) package now exists. Install: diff --git a/docs/varnish/vcl/varnish4.vcl b/docs/varnish/vcl/varnish4.vcl index 03d63bda..01355b7b 100644 --- a/docs/varnish/vcl/varnish4.vcl +++ b/docs/varnish/vcl/varnish4.vcl @@ -1,6 +1,6 @@ // Varnish VCL for: // - Varnish 4.1 or higher (4.1LTS or 6.0LTS recommended, and is what we mainly test against) -// - Varnish xkey vmod (via varnish-modules package, or via Varnish Plus) +// - Varnish xkey vmod (via varnish-modules package 0.10.2 or higher, or via Varnish Plus) // - eZ Platform 1.13 with ezplatform-http-cache bundle // // Make sure to at least adjust default parameters.yml, defaults there reflect our testing needs with docker. diff --git a/docs/varnish/vcl/varnish5.vcl b/docs/varnish/vcl/varnish5.vcl index bee8f22a..05c6f84b 100644 --- a/docs/varnish/vcl/varnish5.vcl +++ b/docs/varnish/vcl/varnish5.vcl @@ -1,6 +1,6 @@ // Varnish VCL for: // - Varnish 5.1 or higher (6.0LTS recommended, and is what we mainly test against) -// - Varnish xkey vmod (via varnish-modules package, or via Varnish Plus) +// - Varnish xkey vmod (via varnish-modules package 0.10.2 or higher, or via Varnish Plus) // - eZ Platform 2.5LTS or higher with ezplatform-http-cache (this) bundle // // Make sure to at least adjust default parameters.yml, defaults there reflect our testing needs with docker. diff --git a/src/PurgeClient/VarnishPurgeClient.php b/src/PurgeClient/VarnishPurgeClient.php index caafd135..f71449a0 100644 --- a/src/PurgeClient/VarnishPurgeClient.php +++ b/src/PurgeClient/VarnishPurgeClient.php @@ -16,6 +16,8 @@ class VarnishPurgeClient implements PurgeClientInterface { const INVALIDATE_TOKEN_PARAM = 'http_cache.varnish_invalidate_token'; const INVALIDATE_TOKEN_PARAM_NAME = 'x-invalidate-token'; + const DEFAULT_HEADER_LENGTH = 7500; + const XKEY_TAG_SEPERATOR = ' '; /** * @var \FOS\HttpCacheBundle\CacheManager @@ -44,20 +46,19 @@ public function purge($tags) return; } - // As key only support one tag being invalidated at a time, we loop. - // These will be queued by FOS\HttpCache\ProxyClient\Varnish and handled on kernel.terminate. - foreach (array_unique((array)$tags) as $tag) { - if (is_numeric($tag)) { - $tag = 'location-' . $tag; - } + // For 5.4/1.x BC make sure to map any int to location id tag + $tags = array_unique(array_map(static function ($tag) { + return is_numeric($tag) ? 'location-' . $tag : $tag; + }, + (array)$tags + )); - $headers = [ - 'key' => $tag, - 'Host' => empty($_SERVER['SERVER_NAME']) ? parse_url($this->configResolver->getParameter('http_cache.purge_servers')[0], PHP_URL_HOST) : $_SERVER['SERVER_NAME'], - ]; - - $headers = $this->addPurgeAuthHeader($headers); + $headers = $this->getPurgeHeaders(); + $chunkSize = $this->determineTagsPerHeader($tags); + // NB! This requries varnish-modules 0.10.2, if you need support for varnish-modules 0.9.x, use ezplatform-http-cache 0.8.x + foreach (array_chunk($tags, $chunkSize) as $tagchunk) { + $headers['key'] = implode(' ', $tagchunk); $this->cacheManager->invalidatePath( '/', $headers @@ -67,12 +68,8 @@ public function purge($tags) public function purgeAll() { - $headers = [ - 'key' => 'ez-all', - 'Host' => empty($_SERVER['SERVER_NAME']) ? parse_url($this->configResolver->getParameter('http_cache.purge_servers')[0], PHP_URL_HOST) : $_SERVER['SERVER_NAME'], - ]; - - $headers = $this->addPurgeAuthHeader($headers); + $headers = $this->getPurgeHeaders(); + $headers['key'] = 'ez-all'; $this->cacheManager->invalidatePath( '/', @@ -81,13 +78,16 @@ public function purgeAll() } /** - * Adds an Authentication header for Purge. + * Adds an generic headers needed for purge (Host and Authentication). * - * @param array $headers * @return array */ - private function addPurgeAuthHeader(array $headers) + private function getPurgeHeaders() { + $headers = [ + 'Host' => empty($_SERVER['SERVER_NAME']) ? parse_url($this->configResolver->getParameter('http_cache.purge_servers')[0], PHP_URL_HOST) : $_SERVER['SERVER_NAME'], + ]; + if ($this->configResolver->hasParameter(self::INVALIDATE_TOKEN_PARAM) && null !== ($token = $this->configResolver->getParameter(self::INVALIDATE_TOKEN_PARAM)) ) { @@ -96,4 +96,23 @@ private function addPurgeAuthHeader(array $headers) return $headers; } + + /** + * Get amount of tags per header, adapted from FOSHttpCache 2.x. + * + * @param array $tags + * + * @return int + */ + private function determineTagsPerHeader(array $tags) + { + if (mb_strlen(implode(self::XKEY_TAG_SEPERATOR, $tags)) < self::DEFAULT_HEADER_LENGTH) { + return count($tags); + } + + // Estimate the amount of tags by dividing the max header length by the largest tag (minus the glue length) + $tagsize = max(array_map('mb_strlen', $tags)); + + return floor(self::DEFAULT_HEADER_LENGTH / ($tagsize + strlen(self::XKEY_TAG_SEPERATOR))) ?: 1; + } } diff --git a/tests/PurgeClient/VarnishPurgeClientTest.php b/tests/PurgeClient/VarnishPurgeClientTest.php index d1e14159..0a0cea73 100644 --- a/tests/PurgeClient/VarnishPurgeClientTest.php +++ b/tests/PurgeClient/VarnishPurgeClientTest.php @@ -116,30 +116,34 @@ public function testPurgeOneLocationIdWithAuthHeaderAndKey() */ public function testPurge(array $locationIds) { - foreach ($locationIds as $key => $locationId) { - $this->configResolver - ->expects($this->at($key * 3)) - ->method('getParameter') - ->with('http_cache.purge_servers') - ->willReturn(['https://varnishpurgehost']); + $this->configResolver + ->expects($this->at(0)) + ->method('getParameter') + ->with('http_cache.purge_servers') + ->willReturn(['https://varnishpurgehost']); - $this->configResolver - ->expects($this->at($key * 3 + 1)) - ->method('hasParameter') - ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM) - ->willReturn(true); + $this->configResolver + ->expects($this->at(1)) + ->method('hasParameter') + ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM) + ->willReturn(true); - $this->configResolver - ->expects($this->at($key * 3 + 2)) - ->method('getParameter') - ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM) - ->willReturn(null); + $this->configResolver + ->expects($this->at(2)) + ->method('getParameter') + ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM) + ->willReturn(null); + + $keys = array_map(static function ($id) { + return "location-$id"; + }, + $locationIds + ); - $this->cacheManager - ->expects($this->at($key)) - ->method('invalidatePath') - ->with('/', ['key' => "location-$locationId", 'Host' => 'varnishpurgehost']); - } + $this->cacheManager + ->expects($this->once()) + ->method('invalidatePath') + ->with('/', ['key' => implode(' ', $keys), 'Host' => 'varnishpurgehost']); $this->purgeClient->purge($locationIds); } @@ -152,30 +156,34 @@ public function testPurgeWithAuthHeaderAndKey(array $locationIds = []) $tokenName = VarnishPurgeClient::INVALIDATE_TOKEN_PARAM_NAME; $token = 'secret-token-key'; - foreach ($locationIds as $key => $locationId) { - $this->configResolver - ->expects($this->at($key * 3)) - ->method('getParameter') - ->with('http_cache.purge_servers') - ->willReturn(['https://varnishpurgehost']); + $this->configResolver + ->expects($this->at(0)) + ->method('getParameter') + ->with('http_cache.purge_servers') + ->willReturn(['https://varnishpurgehost']); - $this->configResolver - ->expects($this->at($key * 3 + 1)) - ->method('hasParameter') - ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM) - ->willReturn(true); + $this->configResolver + ->expects($this->at(1)) + ->method('hasParameter') + ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM) + ->willReturn(true); - $this->configResolver - ->expects($this->at($key * 3 + 2)) - ->method('getParameter') - ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM) - ->willReturn($token); - - $this->cacheManager - ->expects($this->at($key)) - ->method('invalidatePath') - ->with('/', ['key' => "location-$locationId", 'Host' => 'varnishpurgehost', $tokenName => $token]); - } + $this->configResolver + ->expects($this->at(2)) + ->method('getParameter') + ->with(VarnishPurgeClient::INVALIDATE_TOKEN_PARAM) + ->willReturn($token); + + $keys = array_map(static function ($id) { + return "location-$id"; + }, + $locationIds + ); + + $this->cacheManager + ->expects($this->once()) + ->method('invalidatePath') + ->with('/', ['key' => implode(' ', $keys), 'Host' => 'varnishpurgehost', $tokenName => $token]); $this->purgeClient->purge($locationIds); }