From e8b21f0979c4807b28f7be45aff0d25cca1585ae Mon Sep 17 00:00:00 2001 From: Alexander Akait <4567934+alexander-akait@users.noreply.github.com> Date: Tue, 7 Sep 2021 23:09:38 +0300 Subject: [PATCH] feat: don't read full file if `Range` header is present --- package-lock.json | 36 ++--- src/middleware.js | 192 +++++++++++++++++++------- src/utils/compatibleAPI.js | 89 ++++++++++++ src/utils/handleRangeHeaders.js | 79 ----------- test/middleware.test.js | 117 ++++++++++++++-- test/utils/handleRangeHeaders.test.js | 111 --------------- 6 files changed, 355 insertions(+), 269 deletions(-) create mode 100644 src/utils/compatibleAPI.js delete mode 100644 src/utils/handleRangeHeaders.js delete mode 100644 test/utils/handleRangeHeaders.test.js diff --git a/package-lock.json b/package-lock.json index b65cd6fb4..7017eaa83 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4579,9 +4579,9 @@ "dev": true }, "node_modules/colorette": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.3.0.tgz", - "integrity": "sha512-ecORCqbSFP7Wm8Y6lyqMJjexBQqXSF7SSeaTyGGphogUjBlFP9m9o08wy86HL2uB7fMTxtOUzLMk7ogKcxMg1w==" + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.4.0.tgz", + "integrity": "sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==" }, "node_modules/combined-stream": { "version": "1.0.8", @@ -5665,9 +5665,9 @@ "dev": true }, "node_modules/electron-to-chromium": { - "version": "1.3.830", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.830.tgz", - "integrity": "sha512-gBN7wNAxV5vl1430dG+XRcQhD4pIeYeak6p6rjdCtlz5wWNwDad8jwvphe5oi1chL5MV6RNRikfffBBiFuj+rQ==", + "version": "1.3.831", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.831.tgz", + "integrity": "sha512-0tc2lPzgEipHCyRcvDTTaBk5+jSPfNaCvbQdevNMqJkHLvrBiwhygPR0hDyPZEK7Xztvv+58gSFKJ/AUVT1yYQ==", "dev": true }, "node_modules/emittery": { @@ -14599,9 +14599,9 @@ } }, "node_modules/uglify-js": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.14.1.tgz", - "integrity": "sha512-JhS3hmcVaXlp/xSo3PKY5R0JqKs5M3IV+exdLHW99qKvKivPO4Z8qbej6mte17SOPqAOVMjt/XGgWacnFSzM3g==", + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.14.2.tgz", + "integrity": "sha512-rtPMlmcO4agTUfz10CbgJ1k6UAoXM2gWb3GoMPPZB/+/Ackf8lNWk11K4rYi2D0apgoFRLtQOZhb+/iGNJq26A==", "dev": true, "optional": true, "bin": { @@ -18651,9 +18651,9 @@ "dev": true }, "colorette": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.3.0.tgz", - "integrity": "sha512-ecORCqbSFP7Wm8Y6lyqMJjexBQqXSF7SSeaTyGGphogUjBlFP9m9o08wy86HL2uB7fMTxtOUzLMk7ogKcxMg1w==" + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.4.0.tgz", + "integrity": "sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==" }, "combined-stream": { "version": "1.0.8", @@ -19466,9 +19466,9 @@ "dev": true }, "electron-to-chromium": { - "version": "1.3.830", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.830.tgz", - "integrity": "sha512-gBN7wNAxV5vl1430dG+XRcQhD4pIeYeak6p6rjdCtlz5wWNwDad8jwvphe5oi1chL5MV6RNRikfffBBiFuj+rQ==", + "version": "1.3.831", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.831.tgz", + "integrity": "sha512-0tc2lPzgEipHCyRcvDTTaBk5+jSPfNaCvbQdevNMqJkHLvrBiwhygPR0hDyPZEK7Xztvv+58gSFKJ/AUVT1yYQ==", "dev": true }, "emittery": { @@ -26284,9 +26284,9 @@ } }, "uglify-js": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.14.1.tgz", - "integrity": "sha512-JhS3hmcVaXlp/xSo3PKY5R0JqKs5M3IV+exdLHW99qKvKivPO4Z8qbej6mte17SOPqAOVMjt/XGgWacnFSzM3g==", + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.14.2.tgz", + "integrity": "sha512-rtPMlmcO4agTUfz10CbgJ1k6UAoXM2gWb3GoMPPZB/+/Ackf8lNWk11K4rYi2D0apgoFRLtQOZhb+/iGNJq26A==", "dev": true, "optional": true }, diff --git a/src/middleware.js b/src/middleware.js index 8c63c1a23..c9c8b4f22 100644 --- a/src/middleware.js +++ b/src/middleware.js @@ -1,11 +1,42 @@ import path from "path"; import mime from "mime-types"; +import parseRange from "range-parser"; import getFilenameFromUrl from "./utils/getFilenameFromUrl"; -import handleRangeHeaders from "./utils/handleRangeHeaders"; +import { + getHeaderNames, + getHeaderFromRequest, + getHeaderFromResponse, + setHeaderForResponse, + setStatusCode, + send, +} from "./utils/compatibleAPI"; import ready from "./utils/ready"; +function getValueContentRangeHeader(type, size, range) { + return `${type} ${range ? `${range.start}-${range.end}` : "*"}/${size}`; +} + +function createHtmlDocument(title, body) { + return ( + `${ + "\n" + + '\n' + + "\n" + + '\n' + + "" + }${title}\n` + + `\n` + + `\n` + + `
${body}
\n` + + `\n` + + `\n` + ); +} + +const BYTES_RANGE_REGEXP = /^ *bytes/i; + export default function wrapper(context) { return async function middleware(req, res, next) { const acceptedMethods = context.options.methods || ["GET", "HEAD"]; @@ -16,6 +47,7 @@ export default function wrapper(context) { if (!acceptedMethods.includes(req.method)) { await goNext(); + return; } @@ -42,80 +74,146 @@ export default function wrapper(context) { async function processRequest() { const filename = getFilenameFromUrl(context, req.url); - let { headers } = context.options; - - if (typeof headers === "function") { - headers = headers(req, res, context); - } - - let content; if (!filename) { await goNext(); + return; } - try { - content = context.outputFileSystem.readFileSync(filename); - } catch (_ignoreError) { - await goNext(); - return; + let { headers } = context.options; + + if (typeof headers === "function") { + headers = headers(req, res, context); } - const contentTypeHeader = res.get - ? res.get("Content-Type") - : res.getHeader("Content-Type"); + if (headers) { + const names = Object.keys(headers); + + for (const name of names) { + setHeaderForResponse(res, name, headers[name]); + } + } - if (!contentTypeHeader) { + if (!getHeaderFromResponse(res, "Content-Type")) { // content-type name(like application/javascript; charset=utf-8) or false const contentType = mime.contentType(path.extname(filename)); // Only set content-type header if media type is known // https://tools.ietf.org/html/rfc7231#section-3.1.1.5 if (contentType) { - // Express API - if (res.set) { - res.set("Content-Type", contentType); - } - // Node.js API - else { - res.setHeader("Content-Type", contentType); - } + setHeaderForResponse(res, "Content-Type", contentType); } } - if (headers) { - const names = Object.keys(headers); + if (!getHeaderFromResponse(res, "Accept-Ranges")) { + setHeaderForResponse(res, "Accept-Ranges", "bytes"); + } - for (const name of names) { - // Express API - if (res.set) { - res.set(name, headers[name]); - } - // Node.js API - else { - res.setHeader(name, headers[name]); + const rangeHeader = getHeaderFromRequest(req, "range"); + + let start; + let end; + + if (rangeHeader && BYTES_RANGE_REGEXP.test(rangeHeader)) { + const size = await new Promise((resolve) => { + context.outputFileSystem.lstat(filename, (error, stats) => { + if (error) { + context.logger.error(error); + + return; + } + + resolve(stats.size); + }); + }); + + const parsedRanges = parseRange(size, rangeHeader, { + combine: true, + }); + + if (parsedRanges === -1) { + const message = "Unsatisfiable range for 'Range' header."; + + context.logger.error(message); + + const existingHeaders = getHeaderNames(res); + + for (let i = 0; i < existingHeaders.length; i++) { + res.removeHeader(existingHeaders[i]); } + + setStatusCode(res, 416); + setHeaderForResponse( + res, + "Content-Range", + getValueContentRangeHeader("bytes", size) + ); + setHeaderForResponse(res, "Content-Type", "text/html; charset=utf-8"); + + const document = createHtmlDocument(416, `Error: ${message}`); + const byteLength = Buffer.byteLength(document); + + setHeaderForResponse( + res, + "Content-Length", + Buffer.byteLength(document) + ); + + send(req, res, document, byteLength); + + return; + } else if (parsedRanges === -2) { + context.logger.error( + "A malformed 'Range' header was provided. A regular response will be sent for this request." + ); + } else if (parsedRanges.length > 1) { + context.logger.error( + "A 'Range' header with multiple ranges was provided. Multiple ranges are not supported, so a regular response will be sent for this request." + ); } - } - // Buffer - content = handleRangeHeaders(context, content, req, res); + if (parsedRanges !== -2 && parsedRanges.length === 1) { + // Content-Range + setStatusCode(res, 206); + setHeaderForResponse( + res, + "Content-Range", + getValueContentRangeHeader("bytes", size, parsedRanges[0]) + ); - // Express API - if (res.send) { - res.send(content); + [{ start, end }] = parsedRanges; + } } - // Node.js API - else { - res.setHeader("Content-Length", content.length); - if (req.method === "HEAD") { - res.end(); + const isFsSupportsStream = + typeof context.outputFileSystem.createReadStream === "function"; + + let bufferOtStream; + let byteLength; + + try { + if ( + typeof start !== "undefined" && + typeof end !== "undefined" && + isFsSupportsStream + ) { + bufferOtStream = context.outputFileSystem.createReadStream(filename, { + start, + end, + }); + byteLength = end - start + 1; } else { - res.end(content); + bufferOtStream = context.outputFileSystem.readFileSync(filename); + byteLength = Buffer.byteLength(bufferOtStream); } + } catch (_ignoreError) { + await goNext(); + + return; } + + send(req, res, bufferOtStream, byteLength); } }; } diff --git a/src/utils/compatibleAPI.js b/src/utils/compatibleAPI.js new file mode 100644 index 000000000..f5eef0ea2 --- /dev/null +++ b/src/utils/compatibleAPI.js @@ -0,0 +1,89 @@ +function getHeaderNames(res) { + return typeof res.getHeaderNames !== "function" + ? // eslint-disable-next-line no-underscore-dangle + Object.keys(res._headers || {}) + : res.getHeaderNames(); +} + +function getHeaderFromRequest(req, name) { + // Express API + if (typeof req.get === "function") { + return req.get("range"); + } + + // Node.js API + return req.headers[name]; +} + +function getHeaderFromResponse(res, name) { + // Express API + if (typeof res.get === "function") { + return res.get(name); + } + + // Node.js API + return res.getHeader(name); +} + +function setHeaderForResponse(res, name, value) { + // Express API + if (typeof res.set === "function") { + res.set(name, value); + + return; + } + + // Node.js API + res.setHeader(name, value); +} + +function setStatusCode(res, code) { + if (typeof res.status === "function") { + res.status(code); + + return; + } + + // eslint-disable-next-line no-param-reassign + res.statusCode = code; +} + +function send(req, res, bufferOtStream, byteLength) { + if (typeof bufferOtStream.pipe === "function") { + setHeaderForResponse(res, "Content-Length", byteLength); + + if (req.method === "HEAD") { + res.end(); + + return; + } + + bufferOtStream.pipe(res); + + return; + } + + if (typeof res.send === "function") { + res.send(bufferOtStream); + + return; + } + + // Only Node.js API used + res.setHeader("Content-Length", byteLength); + + if (req.method === "HEAD") { + res.end(); + } else { + res.end(bufferOtStream); + } +} + +module.exports = { + getHeaderNames, + getHeaderFromRequest, + getHeaderFromResponse, + setHeaderForResponse, + setStatusCode, + send, +}; diff --git a/src/utils/handleRangeHeaders.js b/src/utils/handleRangeHeaders.js deleted file mode 100644 index 06514a437..000000000 --- a/src/utils/handleRangeHeaders.js +++ /dev/null @@ -1,79 +0,0 @@ -import parseRange from "range-parser"; - -export default function handleRangeHeaders(context, content, req, res) { - // assumes express API. For other servers, need to add logic to access - // alternative header APIs - if (res.set) { - res.set("Accept-Ranges", "bytes"); - } else { - res.setHeader("Accept-Ranges", "bytes"); - } - - let range; - - // Express API - if (req.get) { - range = req.get("range"); - } - // Node.js API - else { - ({ range } = req.headers); - } - - if (range) { - const ranges = parseRange(content.length, range); - - // unsatisfiable - if (ranges === -1) { - // Express API - if (res.set) { - res.set("Content-Range", `bytes */${content.length}`); - res.status(416); - } - // Node.js API - else { - // eslint-disable-next-line no-param-reassign - res.statusCode = 416; - res.setHeader("Content-Range", `bytes */${content.length}`); - } - } else if (ranges === -2) { - // malformed header treated as regular response - context.logger.error( - "A malformed Range header was provided. A regular response will be sent for this request." - ); - } else if (ranges.length !== 1) { - // multiple ranges treated as regular response - context.logger.error( - "A Range header with multiple ranges was provided. Multiple ranges are not supported, so a regular response will be sent for this request." - ); - } else { - // valid range header - const { length } = content; - - // Express API - if (res.set) { - // Content-Range - res.status(206); - res.set( - "Content-Range", - `bytes ${ranges[0].start}-${ranges[0].end}/${length}` - ); - } - // Node.js API - else { - // Content-Range - // eslint-disable-next-line no-param-reassign - res.statusCode = 206; - res.setHeader( - "Content-Range", - `bytes ${ranges[0].start}-${ranges[0].end}/${length}` - ); - } - - // eslint-disable-next-line no-param-reassign - content = content.slice(ranges[0].start, ranges[0].end + 1); - } - } - - return content; -} diff --git a/test/middleware.test.js b/test/middleware.test.js index 024b6de34..adbc24434 100644 --- a/test/middleware.test.js +++ b/test/middleware.test.js @@ -64,6 +64,7 @@ describe.each([ describe("basic", () => { describe("should work", () => { let compiler; + let codeContent; let codeLength; const outputPath = path.resolve(__dirname, "./outputs/basic-test"); @@ -84,7 +85,9 @@ describe.each([ listen = listenShorthand(() => { compiler.hooks.afterCompile.tap("wdm-test", (params) => { - codeLength = params.assets["bundle.js"].source().length; + codeContent = params.assets["bundle.js"].source(); + codeLength = Buffer.byteLength(codeContent); + done(); }); }); @@ -134,26 +137,17 @@ describe.each([ }); it('should return the "200" code for the "GET" request to the bundle file', (done) => { - const fileData = instance.context.outputFileSystem.readFileSync( - path.resolve(outputPath, "bundle.js") - ); - request(app) .get("/bundle.js") - .expect("Content-Length", fileData.byteLength.toString()) + .expect("Content-Length", String(codeLength)) .expect("Content-Type", "application/javascript; charset=utf-8") - .expect(200, fileData.toString(), done); + .expect(200, codeContent, done); }); it('should return the "200" code for the "HEAD" request to the bundle file', (done) => { request(app) .head("/bundle.js") - .expect( - "Content-Length", - instance.context.outputFileSystem - .readFileSync(path.resolve(outputPath, "bundle.js")) - .byteLength.toString() - ) + .expect("Content-Length", String(codeLength)) .expect("Content-Type", "application/javascript; charset=utf-8") // eslint-disable-next-line no-undefined .expect(200, undefined, done); @@ -227,6 +221,8 @@ describe.each([ request(app) .get("/bundle.js") .set("Range", "bytes=9999999-") + .expect("Content-Type", "text/html; charset=utf-8") + .expect("Content-Range", `bytes */${codeLength}`) .expect(416, done); }); @@ -235,14 +231,107 @@ describe.each([ .get("/bundle.js") .set("Range", "bytes=3000-3500") .expect("Content-Length", "501") + .expect("Content-Type", "application/javascript; charset=utf-8") + .expect("Content-Range", `bytes 3000-3500/${codeLength}`) + .expect(206) + .then((response) => { + expect(response.text).toBe(codeContent.substr(3000, 501)); + expect(response.text.length).toBe(501); + + done(); + }); + }); + + it('should return the "206" code for the "GET" request with the valid range header for "HEAD" request', (done) => { + request(app) + .head("/bundle.js") + .set("Range", "bytes=3000-3500") + .expect("Content-Length", "501") + .expect("Content-Type", "application/javascript; charset=utf-8") + .expect("Content-Range", `bytes 3000-3500/${codeLength}`) + .expect(206) + .then((response) => { + expect(response.text).toBeUndefined(); + + done(); + }); + }); + + it('should return the "206" code for the "GET" request with the valid range header (lowercase)', (done) => { + request(app) + .get("/bundle.js") + .set("range", "bytes=3000-3500") + .expect("Content-Length", "501") + .expect("Content-Type", "application/javascript; charset=utf-8") + .expect("Content-Range", `bytes 3000-3500/${codeLength}`) + .expect(206) + .then((response) => { + expect(response.text).toBe(codeContent.substr(3000, 501)); + expect(response.text.length).toBe(501); + + done(); + }); + }); + + it('should return the "206" code for the "GET" request with the valid range header (uppercase)', (done) => { + request(app) + .get("/bundle.js") + .set("RANGE", "BYTES=3000-3500") + .expect("Content-Length", "501") + .expect("Content-Type", "application/javascript; charset=utf-8") .expect("Content-Range", `bytes 3000-3500/${codeLength}`) - .expect(206, done); + .expect(206) + .then((response) => { + expect(response.text).toBe(codeContent.substr(3000, 501)); + expect(response.text.length).toBe(501); + + done(); + }); + }); + + it('should return the "206" code for the "GET" request with the valid range header when range starts with 0', (done) => { + request(app) + .get("/bundle.js") + .set("Range", "bytes=0-3500") + .expect("Content-Length", "3501") + .expect("Content-Type", "application/javascript; charset=utf-8") + .expect("Content-Range", `bytes 0-3500/${codeLength}`) + .expect(206) + .then((response) => { + expect(response.text).toBe(codeContent.substr(0, 3501)); + expect(response.text.length).toBe(3501); + + done(); + }); + }); + + it('should return the "206" code for the "GET" request with the valid range header with multiple values', (done) => { + request(app) + .get("/bundle.js") + .set("Range", "bytes=0-499, 499-800") + .expect("Content-Length", "801") + .expect("Content-Type", "application/javascript; charset=utf-8") + .expect("Content-Range", `bytes 0-800/${codeLength}`) + .expect(206) + .then((response) => { + expect(response.text).toBe(codeContent.substr(0, 801)); + expect(response.text.length).toBe(801); + + done(); + }); }); it('should return the "200" code for the "GET" request with malformed range header which is ignored', (done) => { request(app).get("/bundle.js").set("Range", "abc").expect(200, done); }); + it('should return the "200" code for the "GET" request with malformed range header which is ignored #2', (done) => { + request(app) + .get("/bundle.js") + .set("Range", "bytes") + .expect(200, done); + }); + it('should return the "200" code for the "GET" request with multiple range header which is ignored', (done) => { request(app) .get("/bundle.js") diff --git a/test/utils/handleRangeHeaders.test.js b/test/utils/handleRangeHeaders.test.js deleted file mode 100644 index 8aa3dcdb0..000000000 --- a/test/utils/handleRangeHeaders.test.js +++ /dev/null @@ -1,111 +0,0 @@ -import handleRangeHeaders from "../../src/utils/handleRangeHeaders"; - -describe("handleRangeHeaders", () => { - let context; - - beforeEach(() => { - context = { - logger: { - error: jest.fn(), - }, - }; - }); - - it("should return content in range with valid range header", () => { - const content = "abcdef"; - const req = { - headers: { - range: "bytes=1-4", - }, - get(field) { - return this.headers[field]; - }, - }; - - const res = { - set: jest.fn(), - status(statusCode) { - this.statusCode = statusCode; - }, - }; - - const contentRes = handleRangeHeaders(context, content, req, res); - expect(contentRes).toEqual("bcde"); - expect(res.statusCode).toEqual(206); - expect(res.set.mock.calls).toMatchSnapshot(); - }); - - it("should handle malformed range header", () => { - const content = "abcdef"; - const req = { - headers: { - range: "abc", - }, - get(field) { - return this.headers[field]; - }, - }; - - const res = { - set: jest.fn(), - status(statusCode) { - this.statusCode = statusCode; - }, - }; - - const contentRes = handleRangeHeaders(context, content, req, res); - expect(contentRes).toEqual("abcdef"); - expect(context.logger.error.mock.calls).toMatchSnapshot(); - expect(res.statusCode).toBeUndefined(); - expect(res.set.mock.calls).toMatchSnapshot(); - }); - - it("should handle unsatisfiable range", () => { - const content = "abcdef"; - const req = { - headers: { - range: "bytes=10-20", - }, - get(field) { - return this.headers[field]; - }, - }; - - const res = { - set: jest.fn(), - status(statusCode) { - this.statusCode = statusCode; - }, - }; - - const contentRes = handleRangeHeaders(context, content, req, res); - expect(contentRes).toEqual("abcdef"); - expect(res.statusCode).toEqual(416); - expect(res.set.mock.calls).toMatchSnapshot(); - }); - - it("should handle multiple ranges", () => { - const content = "abcdef"; - const req = { - headers: { - range: "bytes=1-2,4-5", - }, - get(field) { - return this.headers[field]; - }, - }; - - const res = { - set: jest.fn(), - status(statusCode) { - this.statusCode = statusCode; - }, - }; - - const contentRes = handleRangeHeaders(context, content, req, res); - expect(contentRes).toEqual("abcdef"); - expect(context.logger.error.mock.calls).toMatchSnapshot(); - expect(res.statusCode).toBeUndefined(); - expect(res.set.mock.calls).toMatchSnapshot(); - }); -});