diff --git a/middleware/robots.js b/middleware/robots.js index b50d8afe8e35..f36eaa833b0b 100644 --- a/middleware/robots.js +++ b/middleware/robots.js @@ -1,5 +1,6 @@ const languages = require('../lib/languages') const products = require('../lib/all-products') +const { deprecated } = require('../lib/enterprise-server-releases.js') let defaultResponse = 'User-agent: *' @@ -34,5 +35,13 @@ module.exports = function (req, res, next) { defaultResponse = defaultResponse.concat(`\nDisallow: /*${product.href}\nDisallow: /*/enterprise/*/user${product.href}`) }) + // Disallow crawling of Deprecated enterprise versions + deprecated + .forEach(version => { + defaultResponse = defaultResponse + .concat(`\nDisallow: /*/enterprise-server@${version}/*`) + .concat(`\nDisallow: /*/enterprise/${version}/*`) + }) + return res.send(defaultResponse) } diff --git a/tests/rendering/robots-txt.js b/tests/rendering/robots-txt.js index f070f0e0a80f..eb41c64e190b 100644 --- a/tests/rendering/robots-txt.js +++ b/tests/rendering/robots-txt.js @@ -89,4 +89,25 @@ describe('robots.txt', () => { expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.latest}/user/actions`)).toBe(true) expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user/actions`)).toBe(true) }) + + it('disallows indexing of deprecated enterprise releases', async () => { + enterpriseServerReleases.deprecated.forEach(version => { + const blockedPaths = [ + // English + `https://help.github.com/en/enterprise-server@${version}/actions`, + `https://help.github.com/en/enterprise/${version}/actions`, + `https://help.github.com/en/enterprise-server@${version}/actions/overview`, + `https://help.github.com/en/enterprise/${version}/actions/overview`, + // Japanese + `https://help.github.com/ja/enterprise-server@${version}/actions`, + `https://help.github.com/ja/enterprise/${version}/actions`, + `https://help.github.com/ja/enterprise-server@${version}/actions/overview`, + `https://help.github.com/ja/enterprise/${version}/actions/overview` + ] + + blockedPaths.forEach(path => { + expect(robots.isAllowed(path)).toBe(false) + }) + }) + }) })