From 7924aea2f86d6fb15b188b93251e85feb5106219 Mon Sep 17 00:00:00 2001 From: Chiedo John <2156688+chiedo@users.noreply.github.com> Date: Tue, 3 Nov 2020 08:23:22 -0500 Subject: [PATCH] Instruct search engines to not crawl archived versions (#16292) * Instruct search engines to not crawl archived versions * Update middleware/robots.js Co-authored-by: Sarah Schneider * Add test * Fix spooky bug Co-authored-by: Chiedo Co-authored-by: Sarah Schneider --- middleware/robots.js | 9 +++++++++ tests/rendering/robots-txt.js | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/middleware/robots.js b/middleware/robots.js index b50d8afe8e35..f36eaa833b0b 100644 --- a/middleware/robots.js +++ b/middleware/robots.js @@ -1,5 +1,6 @@ const languages = require('../lib/languages') const products = require('../lib/all-products') +const { deprecated } = require('../lib/enterprise-server-releases.js') let defaultResponse = 'User-agent: *' @@ -34,5 +35,13 @@ module.exports = function (req, res, next) { defaultResponse = defaultResponse.concat(`\nDisallow: /*${product.href}\nDisallow: /*/enterprise/*/user${product.href}`) }) + // Disallow crawling of Deprecated enterprise versions + deprecated + .forEach(version => { + defaultResponse = defaultResponse + .concat(`\nDisallow: /*/enterprise-server@${version}/*`) + .concat(`\nDisallow: /*/enterprise/${version}/*`) + }) + return res.send(defaultResponse) } diff --git a/tests/rendering/robots-txt.js b/tests/rendering/robots-txt.js index f070f0e0a80f..eb41c64e190b 100644 --- a/tests/rendering/robots-txt.js +++ b/tests/rendering/robots-txt.js @@ -89,4 +89,25 @@ describe('robots.txt', () => { expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.latest}/user/actions`)).toBe(true) expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user/actions`)).toBe(true) }) + + it('disallows indexing of deprecated enterprise releases', async () => { + enterpriseServerReleases.deprecated.forEach(version => { + const blockedPaths = [ + // English + `https://help.github.com/en/enterprise-server@${version}/actions`, + `https://help.github.com/en/enterprise/${version}/actions`, + `https://help.github.com/en/enterprise-server@${version}/actions/overview`, + `https://help.github.com/en/enterprise/${version}/actions/overview`, + // Japanese + `https://help.github.com/ja/enterprise-server@${version}/actions`, + `https://help.github.com/ja/enterprise/${version}/actions`, + `https://help.github.com/ja/enterprise-server@${version}/actions/overview`, + `https://help.github.com/ja/enterprise/${version}/actions/overview` + ] + + blockedPaths.forEach(path => { + expect(robots.isAllowed(path)).toBe(false) + }) + }) + }) })