From 301aedbddbd52e8bbc46cf9ef20b0343b9af7f72 Mon Sep 17 00:00:00 2001 From: Addy Osmani Date: Sat, 18 Apr 2020 18:43:45 -0700 Subject: [PATCH] [core] Introduce prefetch chunks build (#171) * Adds prefetch chunks implementation * [fix] unit tests for prefetchchunks (#168) * [fix] unit tests for prefetchchunks * [chore] cleaned up console logs * [infra] default accessor argument added * [fix] mistyped expected value (#169) * [fix] mistyped expected value * [chore] debugging prefetch chunks test * [chore] debugging with normal JS functions * [chore] debugging by adding a real css file * [chore] debugging * [chore] debugging * [chore] debugging * [chore] debugging * [chore] debugging * [chore] debugging * [chore] debugging * [chore] debugging * [chore] debugging tests * [chore] debugging tests * [chore] cleaned up Co-authored-by: Anton Karlovskiy --- .gitignore | 1 + package.json | 7 +- src/chunks.mjs | 141 +++++++++++++++++++++++++++++++++ test/bootstrap.js | 2 +- test/quicklink.spec.js | 31 +++++++- test/rmanifest.json | 37 +++++++++ test/test-prefetch-chunks.html | 59 ++++++++++++++ yarn.lock | 12 +++ 8 files changed, 284 insertions(+), 6 deletions(-) create mode 100644 src/chunks.mjs create mode 100644 test/rmanifest.json create mode 100644 test/test-prefetch-chunks.html diff --git a/.gitignore b/.gitignore index 25e53d46..c20ad2d4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ npm-debug.log* yarn-debug.log* yarn-error.log* dist +.DS_Store # Runtime data pids diff --git a/package.json b/package.json index 846a1ed3..fc924dcc 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,10 @@ "lint": "eslint src/*.mjs test/*.js demos/*.js", "lint-fix": "eslint src/*.mjs test/*.js --fix demos/*.js", "start": "http-server .", - "test": "yarn run build && mocha test/bootstrap.js --recursive test", + "test": "yarn run build-all && mocha test/bootstrap.js --recursive test", "build": "microbundle src/index.mjs --no-sourcemap --external none", + "build-plugin": "microbundle src/chunks.mjs --no-sourcemap --external none -o dist/chunks", + "build-all": "yarn run build && yarn run build-plugin", "prepare": "yarn run -s build", "bundlesize": "bundlesize", "changelog": "yarn conventional-changelog -i CHANGELOG.md -s -r 0", @@ -48,7 +50,8 @@ "lodash": "^4.17.11", "microbundle": "0.11.0", "mocha": "^6.2.2", - "puppeteer": "^2.0.0" + "puppeteer": "^2.0.0", + "route-manifest": "^1.0.0" }, "bundlesize": [ { diff --git a/src/chunks.mjs b/src/chunks.mjs new file mode 100644 index 00000000..5d617e43 --- /dev/null +++ b/src/chunks.mjs @@ -0,0 +1,141 @@ +/** + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +**/ +import throttle from 'throttles'; +import { priority, supported } from './prefetch.mjs'; +import requestIdleCallback from './request-idle-callback.mjs'; + +// Cache of URLs we've prefetched +// Its `size` is compared against `opts.limit` value. +const toPrefetch = new Set(); + +/** + * Determine if the anchor tag should be prefetched. + * A filter can be a RegExp, Function, or Array of both. + * - Function receives `node.href, node` arguments + * - RegExp receives `node.href` only (the full URL) + * @param {Element} node The anchor () tag. + * @param {Mixed} filter The custom filter(s) + * @return {Boolean} If true, then it should be ignored + */ +function isIgnored(node, filter) { + return Array.isArray(filter) + ? filter.some(x => isIgnored(node, x)) + : (filter.test || filter).call(filter, node.href, node); +} + +/** + * Prefetch an array of URLs if the user's effective + * connection type and data-saver preferences suggests + * it would be useful. By default, looks at in-viewport + * links for `document`. Can also work off a supplied + * DOM element or static array of URLs. + * @param {Object} options - Configuration options for quicklink + * @param {Object} [options.el] - DOM element to prefetch in-viewport links of + * @param {Boolean} [options.priority] - Attempt higher priority fetch (low or high) + * @param {Array} [options.origins] - Allowed origins to prefetch (empty allows all) + * @param {Array|RegExp|Function} [options.ignores] - Custom filter(s) that run after origin checks + * @param {Number} [options.timeout] - Timeout after which prefetching will occur + * @param {Number} [options.throttle] - The concurrency limit for prefetching + * @param {Number} [options.limit] - The total number of prefetches to allow + * @param {Function} [options.timeoutFn] - Custom timeout function + * @param {Function} [options.onError] - Error handler for failed `prefetch` requests + * @param {Function} [options.prefetchChunks] - Function to prefetch chunks for route URLs (with route manifest for URL mapping) + */ +export function listen(options) { + if (!options) options = {}; + if (!window.IntersectionObserver) return; + + const [toAdd, isDone] = throttle(options.throttle || 1/0); + const limit = options.limit || 1/0; + + const allowed = options.origins || [location.hostname]; + const ignores = options.ignores || []; + + const timeoutFn = options.timeoutFn || requestIdleCallback; + + const prefetchChunks = options.prefetchChunks; + + const prefetchHandler = urls => { + prefetch(urls, options.priority).then(isDone).catch(err => { + isDone(); if (options.onError) options.onError(err); + }); + }; + + const observer = new IntersectionObserver(entries => { + entries.forEach(entry => { + if (entry.isIntersecting) { + observer.unobserve(entry = entry.target); + // Do not prefetch if will match/exceed limit + if (toPrefetch.size < limit) { + toAdd(() => { + prefetchChunks ? prefetchChunks(entry, prefetchHandler) : prefetchHandler(entry.href); + }); + } + } + }); + }); + + timeoutFn(() => { + // Find all links & Connect them to IO if allowed + (options.el || document).querySelectorAll('a').forEach(link => { + // If the anchor matches a permitted origin + // ~> A `[]` or `true` means everything is allowed + if (!allowed.length || allowed.includes(link.hostname)) { + // If there are any filters, the link must not match any of them + isIgnored(link, ignores) || observer.observe(link); + } + }); + }, { + timeout: options.timeout || 2000 + }); + + return function () { + // wipe url list + toPrefetch.clear(); + // detach IO entries + observer.disconnect(); + }; +} + + +/** +* Prefetch a given URL with an optional preferred fetch priority +* @param {String} url - the URL to fetch +* @param {Boolean} [isPriority] - if is "high" priority +* @param {Object} [conn] - navigator.connection (internal) +* @return {Object} a Promise +*/ +export function prefetch(url, isPriority, conn) { + if (conn = navigator.connection) { + // Don't prefetch if using 2G or if Save-Data is enabled. + if (conn.saveData || /2g/.test(conn.effectiveType)) return; + } + + // Dev must supply own catch() + return Promise.all( + [].concat(url).map(str => { + if (!toPrefetch.has(str)) { + // Add it now, regardless of its success + // ~> so that we don't repeat broken links + toPrefetch.add(str); + + return (isPriority ? priority : supported)( + new URL(str, location.href).toString() + ); + } + }) + ); +} \ No newline at end of file diff --git a/test/bootstrap.js b/test/bootstrap.js index e95220a1..aeb2b172 100644 --- a/test/bootstrap.js +++ b/test/bootstrap.js @@ -18,7 +18,7 @@ before(async function () { // close browser and reset global variables after(function () { - browser.close(); + global.browser.close(); global.browser = globalVariables.browser; global.expect = globalVariables.expect; diff --git a/test/quicklink.spec.js b/test/quicklink.spec.js index 08ce95a5..dffd1db3 100644 --- a/test/quicklink.spec.js +++ b/test/quicklink.spec.js @@ -1,5 +1,6 @@ describe('quicklink tests', function () { - const server = `http://127.0.0.1:8080/test`; + const host = 'http://127.0.0.1:8080'; + const server = `${host}/test`; let page; before(async function () { @@ -194,7 +195,7 @@ describe('quicklink tests', function () { // don't care about first 4 URLs (markup) const ours = responseURLs.slice(4); - expect(ours.length).to.equal(2); + expect(ours.length).to.equal(1); expect(ours).to.include(`${server}/2.html`); }); @@ -222,7 +223,7 @@ describe('quicklink tests', function () { // don't care about first 4 URLs (markup) const ours = responseURLs.slice(4); - expect(ours.length).to.equal(2); + expect(ours.length).to.equal(1); expect(ours).to.include(`${server}/1.html`); }); @@ -255,4 +256,28 @@ describe('quicklink tests', function () { await page.waitFor(250); expect(URLs.length).to.equal(4); }); + + it('should prefetch chunks for in-viewport links', async function () { + const responseURLs = []; + page.on('response', resp => { + responseURLs.push(resp.url()); + }); + await page.goto(`${server}/test-prefetch-chunks.html`); + await page.waitFor(1000); + expect(responseURLs).to.be.an('array'); + // should prefetch chunk URLs for /, /blog and /about links + expect(responseURLs).to.include(`${host}/test/static/css/home.6d953f22.chunk.css`); + expect(responseURLs).to.include(`${host}/test/static/js/home.14835906.chunk.js`); + expect(responseURLs).to.include(`${host}/test/static/media/video.b9b6e9e1.svg`); + expect(responseURLs).to.include(`${host}/test/static/css/blog.2a8b6ab6.chunk.css`); + expect(responseURLs).to.include(`${host}/test/static/js/blog.1dcce8a6.chunk.js`); + expect(responseURLs).to.include(`${host}/test/static/css/about.00ec0d84.chunk.css`); + expect(responseURLs).to.include(`${host}/test/static/js/about.921ebc84.chunk.js`); + // should not prefetch /, /blog and /about links + expect(responseURLs).to.not.include(`${server}`); + expect(responseURLs).to.not.include(`${server}/blog`); + expect(responseURLs).to.not.include(`${server}/about`); + // should prefetch regular links + expect(responseURLs).to.include(`${server}/main.css`); + }); }); diff --git a/test/rmanifest.json b/test/rmanifest.json new file mode 100644 index 00000000..6cad8b1c --- /dev/null +++ b/test/rmanifest.json @@ -0,0 +1,37 @@ +{ + "/about": [{ + "type": "style", + "href": "/test/static/css/about.00ec0d84.chunk.css" + }, { + "type": "script", + "href": "/test/static/js/about.921ebc84.chunk.js" + }], + "/blog": [{ + "type": "style", + "href": "/test/static/css/blog.2a8b6ab6.chunk.css" + }, { + "type": "script", + "href": "/test/static/js/blog.1dcce8a6.chunk.js" + }], + "/": [{ + "type": "style", + "href": "/test/static/css/home.6d953f22.chunk.css" + }, { + "type": "script", + "href": "/test/static/js/home.14835906.chunk.js" + }, { + "type": "image", + "href": "/test/static/media/video.b9b6e9e1.svg" + }], + "/blog/:title": [{ + "type": "style", + "href": "/test/static/css/article.cb6f97df.chunk.css" + }, { + "type": "script", + "href": "/test/static/js/article.cb6f97df.chunk.js" + }], + "*": [{ + "type": "script", + "href": "/test/static/js/6.7f61b1a1.chunk.js" + }] +} \ No newline at end of file diff --git a/test/test-prefetch-chunks.html b/test/test-prefetch-chunks.html new file mode 100644 index 00000000..19744c57 --- /dev/null +++ b/test/test-prefetch-chunks.html @@ -0,0 +1,59 @@ + + + + + + Prefetch: Chunk URL list + + + + + + + Home + Blog + About +
+ CSS +
+ Link 4 + + + + + \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index f8222175..c9c0d3b6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4450,6 +4450,11 @@ regex-cache@^0.4.2: dependencies: is-equal-shallow "^0.1.3" +regexparam@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-1.3.0.tgz#2fe42c93e32a40eff6235d635e0ffa344b92965f" + integrity sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g== + regexpp@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-2.0.1.tgz#8d19d31cf632482b589049f8281f93dbcba4d07f" @@ -4730,6 +4735,13 @@ rollup@^0.67.3: "@types/estree" "0.0.39" "@types/node" "*" +route-manifest@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/route-manifest/-/route-manifest-1.0.0.tgz#0155513f3cd158c18827413845ab1a8ec2ad15e1" + integrity sha512-qn0xJr4nnF4caj0erOLLAHYiNyzqhzpUbgDQcEHrmBoG4sWCDLnIXLH7VccNSxe9cWgbP2Kw/OjME+eH3CeRSA== + dependencies: + regexparam "^1.3.0" + run-async@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.3.0.tgz#0371ab4ae0bdd720d4166d7dfda64ff7a445a6c0"