From fef26a9e0bd01dab8c0da71c72328f0100c7720c Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Wed, 16 Nov 2022 20:51:52 +0100 Subject: [PATCH 01/26] chore(build,content,server): allow mp3/mp4/ogg/ttf/webm attachments Allows these files to be used in live samples. --- build/check-images.ts | 12 ++++++------ build/flaws/broken-links.ts | 4 ++-- build/index.ts | 6 +++--- build/utils.ts | 8 ++++---- content/{image.ts => file-attachment.ts} | 23 ++++++++++++++++++++++- content/index.ts | 2 +- server/index.ts | 8 ++++---- 7 files changed, 42 insertions(+), 21 deletions(-) rename content/{image.ts => file-attachment.ts} (78%) diff --git a/build/check-images.ts b/build/check-images.ts index d7f1f3ec081c..806a6c8571e0 100644 --- a/build/check-images.ts +++ b/build/check-images.ts @@ -6,7 +6,7 @@ import path from "node:path"; import imagesize from "image-size"; -import { Document, Image } from "../content/index.js"; +import { Document, FileAttachment } from "../content/index.js"; import { FLAW_LEVELS, DEFAULT_LOCALE } from "../libs/constants/index.js"; import { findMatchesInText } from "./matches-in-text.js"; import * as cheerio from "cheerio"; @@ -140,12 +140,12 @@ export function checkImageReferences( // but all our images are going to be static. finalSrc = absoluteURL.pathname; // We can use the `finalSrc` to look up and find the image independent - // of the correct case because `Image.findByURL` operates case + // of the correct case because `FileAttachment.findByURL` operates case // insensitively. - // What follows uses the same algorithm as Image.findByURLWithFallback + // What follows uses the same algorithm as FileAttachment.findByURLWithFallback // but only adds a filePath if it exists for the DEFAULT_LOCALE - const filePath = Image.findByURL(finalSrc); + const filePath = FileAttachment.findByURL(finalSrc); let enUSFallback = false; if ( !filePath && @@ -156,7 +156,7 @@ export function checkImageReferences( new RegExp(`^/${doc.locale}/`, "i"), `/${DEFAULT_LOCALE}/` ); - if (Image.findByURL(enUSFinalSrc)) { + if (FileAttachment.findByURL(enUSFinalSrc)) { // Use the en-US src instead finalSrc = enUSFinalSrc; // Note that this `` value can work if you use the @@ -366,7 +366,7 @@ export function checkImageWidths( ); } } else if (!imgSrc.includes("://") && imgSrc.startsWith("/")) { - const filePath = Image.findByURLWithFallback(imgSrc); + const filePath = FileAttachment.findByURLWithFallback(imgSrc); if (filePath) { const dimensions = sizeOf(filePath); img.attr("width", `${dimensions.width}`); diff --git a/build/flaws/broken-links.ts b/build/flaws/broken-links.ts index edfecb1ab97b..9374a60e4e84 100644 --- a/build/flaws/broken-links.ts +++ b/build/flaws/broken-links.ts @@ -3,7 +3,7 @@ import fs from "node:fs"; import { fromMarkdown } from "mdast-util-from-markdown"; import { visit } from "unist-util-visit"; -import { Document, Redirect, Image } from "../../content/index.js"; +import { Document, Redirect, FileAttachment } from "../../content/index.js"; import { findMatchesInText } from "../matches-in-text.js"; import { DEFAULT_LOCALE, @@ -278,7 +278,7 @@ export function getBrokenLinksFlaws( const found = Document.findByURL(hrefNormalized); if (!found) { // Before we give up, check if it's an image. - if (!Image.findByURLWithFallback(hrefNormalized)) { + if (!FileAttachment.findByURLWithFallback(hrefNormalized)) { // Even if it's a redirect, it's still a flaw, but it'll be nice to // know what it *should* be. const resolved = Redirect.resolve(hrefNormalized); diff --git a/build/index.ts b/build/index.ts index b30f9918cae9..01f9880fc257 100644 --- a/build/index.ts +++ b/build/index.ts @@ -33,7 +33,7 @@ import LANGUAGES_RAW from "../libs/languages/index.js"; import { safeDecodeURIComponent } from "../kumascript/src/api/util.js"; import { wrapTables } from "./wrap-tables.js"; import { - getAdjacentImages, + getAdjacentFileAttachments, injectLoadingLazyAttributes, injectNoTranslate, makeTOC, @@ -382,8 +382,8 @@ export async function buildDocument( // The checkImageReferences() does 2 things. Checks image *references* and // it returns which images it checked. But we'll need to complement any // other images in the folder. - getAdjacentImages(path.dirname(document.fileInfo.path)).forEach((fp) => - fileAttachments.add(fp) + getAdjacentFileAttachments(path.dirname(document.fileInfo.path)).forEach( + (fp) => fileAttachments.add(fp) ); // Check the img tags for possible flaws and possible build-time rewrites diff --git a/build/utils.ts b/build/utils.ts index 9b76c2b8e266..f966a165d159 100644 --- a/build/utils.ts +++ b/build/utils.ts @@ -13,7 +13,7 @@ import { rgPath } from "@vscode/ripgrep"; import sanitizeFilename from "sanitize-filename"; import { VALID_MIME_TYPES } from "../libs/constants/index.js"; -import { Image } from "../content/index.js"; +import { FileAttachment } from "../content/index.js"; import { spawnSync } from "node:child_process"; import { BLOG_ROOT } from "../libs/env/index.js"; @@ -184,14 +184,14 @@ export function splitSections(rawHTML) { * * @param {Document} document */ -export function getAdjacentImages(documentDirectory) { +export function getAdjacentFileAttachments(documentDirectory) { const dirents = fs.readdirSync(documentDirectory, { withFileTypes: true }); return dirents .filter((dirent) => { // This needs to match what we do in filecheck/checker.py return ( !dirent.isDirectory() && - /\.(png|jpeg|jpg|gif|svg|webp)$/i.test(dirent.name) + /\.(mp3|mp4|png|jpeg|jpg|gif|ogg|svg|ttf|webm|webp)$/i.test(dirent.name) ); }) .map((dirent) => path.join(documentDirectory, dirent.name)); @@ -262,7 +262,7 @@ export function postLocalFileLinks($, doc) { // So we'll look-up a lot "false positives" that are not images. // Thankfully, this lookup is fast. const url = `${doc.mdn_url}/${href}`; - const image = Image.findByURLWithFallback(url); + const image = FileAttachment.findByURLWithFallback(url); if (image) { $(element).attr("href", url); } diff --git a/content/image.ts b/content/file-attachment.ts similarity index 78% rename from content/image.ts rename to content/file-attachment.ts index e977b05ca9ce..cf323037ad64 100644 --- a/content/image.ts +++ b/content/file-attachment.ts @@ -9,6 +9,27 @@ import { DEFAULT_LOCALE } from "../libs/constants/index.js"; import { ROOTS } from "../libs/env/index.js"; import { memoize, slugToFolder } from "./utils.js"; +function isFileAttachment(filePath: string) { + return ( + isAudio(filePath) || + isFont(filePath) || + isVideo(filePath) || + isImage(filePath) + ); +} + +function isAudio(filePath) { + return /\.(mp3|ogg)$/i.test(filePath); +} + +function isFont(filePath) { + return /\.(ttf)$/i.test(filePath); +} + +function isVideo(filePath) { + return /\.(mp4|webm)$/i.test(filePath); +} + function isImage(filePath: string) { if (fs.statSync(filePath).isDirectory()) { return false; @@ -37,7 +58,7 @@ function urlToFilePath(url: string) { const find = memoize((relativePath: string) => { return ROOTS.map((root) => path.join(root, relativePath)).find( - (filePath) => fs.existsSync(filePath) && isImage(filePath) + (filePath) => fs.existsSync(filePath) && isFileAttachment(filePath) ); }); diff --git a/content/index.ts b/content/index.ts index 0c41d320022c..18f2b8a4476f 100644 --- a/content/index.ts +++ b/content/index.ts @@ -2,7 +2,7 @@ export * as Document from "./document.js"; export * as Translation from "./translation.js"; export { getPopularities } from "./popularities.js"; export * as Redirect from "./redirect.js"; -export * as Image from "./image.js"; +export * as FileAttachment from "./file-attachment.js"; export { buildURL, memoize, diff --git a/server/index.ts b/server/index.ts index 2751bd9f4f1c..a9c704c31bfd 100644 --- a/server/index.ts +++ b/server/index.ts @@ -17,7 +17,7 @@ import { renderContributorsTxt, } from "../build/index.js"; import { findTranslations } from "../content/translations.js"; -import { Document, Redirect, Image } from "../content/index.js"; +import { Document, Redirect, FileAttachment } from "../content/index.js"; import { CSP_VALUE, DEFAULT_LOCALE } from "../libs/constants/index.js"; import { STATIC_ROOT, @@ -294,12 +294,12 @@ app.get("/*", async (req, res, ...args) => { // TODO: Would be nice to have a list of all supported file extensions // in a constants file. - if (/\.(png|webp|gif|jpe?g|svg)$/.test(req.path)) { - // Remember, Image.findByURLWithFallback() will return the absolute file path + if (/\.(gif|jpe?g|mp3|mp4|png|ogg|svg|ttf|webm|webp)$/.test(req.path)) { + // Remember, FileAttachment.findByURLWithFallback() will return the absolute file path // iff it exists on disk. // Using a "fallback" strategy here so that images embedded in live samples // are resolved if they exist in en-US but not in - const filePath = Image.findByURLWithFallback(req.path); + const filePath = FileAttachment.findByURLWithFallback(req.path); if (filePath) { // The second parameter to `send()` has to be either a full absolute // path or a path that doesn't start with `../` otherwise you'd From 034b2f8d8e67b6a9888fbb02abf386839720a947 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Thu, 17 Nov 2022 13:13:54 +0100 Subject: [PATCH 02/26] chore(file-attachment): type params --- build/utils.ts | 2 +- content/file-attachment.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/utils.ts b/build/utils.ts index f966a165d159..de125d54ab9a 100644 --- a/build/utils.ts +++ b/build/utils.ts @@ -184,7 +184,7 @@ export function splitSections(rawHTML) { * * @param {Document} document */ -export function getAdjacentFileAttachments(documentDirectory) { +export function getAdjacentFileAttachments(documentDirectory: string) { const dirents = fs.readdirSync(documentDirectory, { withFileTypes: true }); return dirents .filter((dirent) => { diff --git a/content/file-attachment.ts b/content/file-attachment.ts index cf323037ad64..c4a28c8822e2 100644 --- a/content/file-attachment.ts +++ b/content/file-attachment.ts @@ -18,15 +18,15 @@ function isFileAttachment(filePath: string) { ); } -function isAudio(filePath) { +function isAudio(filePath: string) { return /\.(mp3|ogg)$/i.test(filePath); } -function isFont(filePath) { +function isFont(filePath: string) { return /\.(ttf)$/i.test(filePath); } -function isVideo(filePath) { +function isVideo(filePath: string) { return /\.(mp4|webm)$/i.test(filePath); } From bd11bf5607cb2eab5e0f6cd33aa253e7c3dba6f1 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Thu, 17 Nov 2022 19:31:35 +0100 Subject: [PATCH 03/26] fix(file-attachment): always check for directories --- content/file-attachment.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/content/file-attachment.ts b/content/file-attachment.ts index c4a28c8822e2..3c227f94b7a1 100644 --- a/content/file-attachment.ts +++ b/content/file-attachment.ts @@ -10,6 +10,10 @@ import { ROOTS } from "../libs/env/index.js"; import { memoize, slugToFolder } from "./utils.js"; function isFileAttachment(filePath: string) { + if (fs.statSync(filePath).isDirectory()) { + return false; + } + return ( isAudio(filePath) || isFont(filePath) || @@ -31,9 +35,6 @@ function isVideo(filePath: string) { } function isImage(filePath: string) { - if (fs.statSync(filePath).isDirectory()) { - return false; - } if (filePath.toLowerCase().endsWith(".svg")) { return isSvg(fs.readFileSync(filePath, "utf-8")); } From 32e84463eb0aca67ee89c05ee584bdf49356ee7f Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Thu, 17 Nov 2022 19:36:29 +0100 Subject: [PATCH 04/26] chore(file-attachment): restrict images to file types in content --- content/file-attachment.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/content/file-attachment.ts b/content/file-attachment.ts index 3c227f94b7a1..b77d310aacb5 100644 --- a/content/file-attachment.ts +++ b/content/file-attachment.ts @@ -35,6 +35,10 @@ function isVideo(filePath: string) { } function isImage(filePath: string) { + if (!/\.(gif|jpe?g|png|svg)$/i.test(filePath)) { + return false; + } + if (filePath.toLowerCase().endsWith(".svg")) { return isSvg(fs.readFileSync(filePath, "utf-8")); } From c05538b5a48f6a8c5085849dc8129681f3348435 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Fri, 18 Nov 2022 10:30:22 +0100 Subject: [PATCH 05/26] chore(file-attachment): support woff/woff2 --- build/utils.ts | 2 +- content/file-attachment.ts | 2 +- server/index.ts | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build/utils.ts b/build/utils.ts index de125d54ab9a..a28e91f84521 100644 --- a/build/utils.ts +++ b/build/utils.ts @@ -191,7 +191,7 @@ export function getAdjacentFileAttachments(documentDirectory: string) { // This needs to match what we do in filecheck/checker.py return ( !dirent.isDirectory() && - /\.(mp3|mp4|png|jpeg|jpg|gif|ogg|svg|ttf|webm|webp)$/i.test(dirent.name) + /\.(mp3|mp4|png|jpeg|jpg|gif|ogg|svg|ttf|webm|webp|woff2?)$/i.test(dirent.name) ); }) .map((dirent) => path.join(documentDirectory, dirent.name)); diff --git a/content/file-attachment.ts b/content/file-attachment.ts index b77d310aacb5..96f36a3fa505 100644 --- a/content/file-attachment.ts +++ b/content/file-attachment.ts @@ -27,7 +27,7 @@ function isAudio(filePath: string) { } function isFont(filePath: string) { - return /\.(ttf)$/i.test(filePath); + return /\.(ttf|woff2?)$/i.test(filePath); } function isVideo(filePath: string) { diff --git a/server/index.ts b/server/index.ts index a9c704c31bfd..81416ab3fd97 100644 --- a/server/index.ts +++ b/server/index.ts @@ -294,7 +294,9 @@ app.get("/*", async (req, res, ...args) => { // TODO: Would be nice to have a list of all supported file extensions // in a constants file. - if (/\.(gif|jpe?g|mp3|mp4|png|ogg|svg|ttf|webm|webp)$/.test(req.path)) { + if ( + /\.(gif|jpe?g|mp3|mp4|png|ogg|svg|ttf|webm|webp|woff2?)$/.test(req.path) + ) { // Remember, FileAttachment.findByURLWithFallback() will return the absolute file path // iff it exists on disk. // Using a "fallback" strategy here so that images embedded in live samples From 6c476730c59654680914dd55953271962365b5bf Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Fri, 18 Nov 2022 12:02:19 +0100 Subject: [PATCH 06/26] chore(build): check file-type of audio/video/font files --- build/cli.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/build/cli.ts b/build/cli.ts index 7374194ce0b8..56b7f0a83450 100644 --- a/build/cli.ts +++ b/build/cli.ts @@ -7,6 +7,7 @@ import zlib from "node:zlib"; import chalk from "chalk"; import cliProgress from "cli-progress"; import caporal from "@caporal/core"; +import { fileTypeFromFile } from "file-type"; import inquirer from "inquirer"; import { Document, slugToFolder, translationsOf } from "../content/index.js"; @@ -213,7 +214,23 @@ async function buildDocuments( } for (const filePath of fileAttachments) { - // We *could* use symlinks instead. But, there's no point :) + // Ensure that binary files contain what their extension indicates. + if (/\.(mp3|mp4|ttf|webm|woff2?)$/i.test(filePath)) { + const ext = filePath.split(".").pop(); + const type = await fileTypeFromFile(filePath); + if (!type) { + throw new Error( + `Failed to detect type of file attachment: ${filePath}` + ); + } + if (ext.toLowerCase() !== type.ext) { + throw new Error( + `Unexpected type '${type.mime}' (*.${ext}) detected for file attachment: ${filePath}.` + ); + } + } + + // We *could* use symlinks instead. But, there's no point : // Yes, a symlink is less disk I/O but it's nominal. fs.copyFileSync(filePath, path.join(outPath, path.basename(filePath))); } From 1979be7adc8022463b4bcc95f9b2f7f5252f09c0 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Fri, 25 Nov 2022 22:15:57 +0100 Subject: [PATCH 07/26] fix(constants): add mp3/mp4/ogg/ttf/webm to VALID_MIME_TYPES --- libs/constants/index.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libs/constants/index.js b/libs/constants/index.js index bf209a9e56bd..94d18d27ccc1 100644 --- a/libs/constants/index.js +++ b/libs/constants/index.js @@ -202,9 +202,19 @@ export const MARKDOWN_FILENAME = "index.md"; // --------- export const VALID_MIME_TYPES = new Set([ + "audio/mp4", + "audio/mpeg", + "audio/ogg", + "audio/webm", + "font/woff", + "font/woff2", "image/png", "image/jpeg", // this is what you get for .jpeg *and* .jpg file extensions "image/gif", + "image/webp", + "video/mp4", + "video/ogg", + "video/webm", ]); export const MAX_COMPRESSION_DIFFERENCE_PERCENTAGE = 25; // percent From 367d0a83cda3160435d86190a2345284d938eceb Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Mon, 28 Nov 2022 15:27:14 +0100 Subject: [PATCH 08/26] refactor(build): move file extension check into filecheck --- build/cli.ts | 19 +------------------ filecheck/checker.ts | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/build/cli.ts b/build/cli.ts index 56b7f0a83450..7374194ce0b8 100644 --- a/build/cli.ts +++ b/build/cli.ts @@ -7,7 +7,6 @@ import zlib from "node:zlib"; import chalk from "chalk"; import cliProgress from "cli-progress"; import caporal from "@caporal/core"; -import { fileTypeFromFile } from "file-type"; import inquirer from "inquirer"; import { Document, slugToFolder, translationsOf } from "../content/index.js"; @@ -214,23 +213,7 @@ async function buildDocuments( } for (const filePath of fileAttachments) { - // Ensure that binary files contain what their extension indicates. - if (/\.(mp3|mp4|ttf|webm|woff2?)$/i.test(filePath)) { - const ext = filePath.split(".").pop(); - const type = await fileTypeFromFile(filePath); - if (!type) { - throw new Error( - `Failed to detect type of file attachment: ${filePath}` - ); - } - if (ext.toLowerCase() !== type.ext) { - throw new Error( - `Unexpected type '${type.mime}' (*.${ext}) detected for file attachment: ${filePath}.` - ); - } - } - - // We *could* use symlinks instead. But, there's no point : + // We *could* use symlinks instead. But, there's no point :) // Yes, a symlink is less disk I/O but it's nominal. fs.copyFileSync(filePath, path.join(outPath, path.basename(filePath))); } diff --git a/filecheck/checker.ts b/filecheck/checker.ts index 81a71498535b..7f3d22101b59 100644 --- a/filecheck/checker.ts +++ b/filecheck/checker.ts @@ -78,6 +78,20 @@ export async function checkFile( throw new Error(`${filePath} is 0 bytes`); } + // Ensure that binary files contain what their extension indicates. + if (/\.(mp3|mp4|ttf|webm|woff2?)$/i.test(filePath)) { + const ext = filePath.split(".").pop(); + const type = await fileTypeFromFile(filePath); + if (!type) { + throw new Error(`Failed to detect type of file attachment: ${filePath}`); + } + if (ext.toLowerCase() !== type.ext) { + throw new Error( + `Unexpected type '${type.mime}' (*.${type.ext}) detected for file attachment: ${filePath}.` + ); + } + } + // FileType can't check for .svg files. // So use special case for files called '*.svg' if (path.extname(filePath) === ".svg") { From 22e1977fb459d248980295214a5094798e887d00 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Mon, 28 Nov 2022 21:49:19 +0100 Subject: [PATCH 09/26] refactor(filecheck): extract checkCompression() --- filecheck/checker.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/filecheck/checker.ts b/filecheck/checker.ts index 7f3d22101b59..6ae60711deb4 100644 --- a/filecheck/checker.ts +++ b/filecheck/checker.ts @@ -191,6 +191,10 @@ export async function checkFile( ); } + await checkCompression(filePath, options); +} + +async function checkCompression(filePath, options) { const tempdir = temporaryDirectory(); const extension = path.extname(filePath).toLowerCase(); try { From ab21cd3b97d8c45f1353350efb27685e67dbe44a Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Mon, 28 Nov 2022 21:49:42 +0100 Subject: [PATCH 10/26] fix(filecheck): ignore compression of other files --- filecheck/checker.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/filecheck/checker.ts b/filecheck/checker.ts index 6ae60711deb4..df2d880cd0e1 100644 --- a/filecheck/checker.ts +++ b/filecheck/checker.ts @@ -207,9 +207,12 @@ async function checkCompression(filePath, options) { plugins.push(imageminGifsicle()); } else if (extension === ".svg") { plugins.push(imageminSvgo()); - } else { - throw new Error(`No plugin for ${extension}`); } + + if (!plugins.length) { + return; + } + const files = await imagemin([filePath], { destination: tempdir, plugins, From 864bbcc9a3ce83280076b55cda505fce8082a788 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Mon, 28 Nov 2022 21:51:54 +0100 Subject: [PATCH 11/26] chore: remove TTF support WOFF2 is preferable, as TTF is uncompressed. --- build/utils.ts | 4 +++- content/file-attachment.ts | 2 +- filecheck/checker.ts | 2 +- server/index.ts | 4 +--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build/utils.ts b/build/utils.ts index a28e91f84521..e0b0fe0bad2f 100644 --- a/build/utils.ts +++ b/build/utils.ts @@ -191,7 +191,9 @@ export function getAdjacentFileAttachments(documentDirectory: string) { // This needs to match what we do in filecheck/checker.py return ( !dirent.isDirectory() && - /\.(mp3|mp4|png|jpeg|jpg|gif|ogg|svg|ttf|webm|webp|woff2?)$/i.test(dirent.name) + /\.(mp3|mp4|png|jpeg|jpg|gif|ogg|svg|webm|webp|woff2?)$/i.test( + dirent.name + ) ); }) .map((dirent) => path.join(documentDirectory, dirent.name)); diff --git a/content/file-attachment.ts b/content/file-attachment.ts index 96f36a3fa505..83b0fd309b6a 100644 --- a/content/file-attachment.ts +++ b/content/file-attachment.ts @@ -27,7 +27,7 @@ function isAudio(filePath: string) { } function isFont(filePath: string) { - return /\.(ttf|woff2?)$/i.test(filePath); + return /\.(woff2?)$/i.test(filePath); } function isVideo(filePath: string) { diff --git a/filecheck/checker.ts b/filecheck/checker.ts index df2d880cd0e1..0bad88e14f6d 100644 --- a/filecheck/checker.ts +++ b/filecheck/checker.ts @@ -79,7 +79,7 @@ export async function checkFile( } // Ensure that binary files contain what their extension indicates. - if (/\.(mp3|mp4|ttf|webm|woff2?)$/i.test(filePath)) { + if (/\.(mp3|mp4|webm|woff2?)$/i.test(filePath)) { const ext = filePath.split(".").pop(); const type = await fileTypeFromFile(filePath); if (!type) { diff --git a/server/index.ts b/server/index.ts index 81416ab3fd97..53da336825e3 100644 --- a/server/index.ts +++ b/server/index.ts @@ -294,9 +294,7 @@ app.get("/*", async (req, res, ...args) => { // TODO: Would be nice to have a list of all supported file extensions // in a constants file. - if ( - /\.(gif|jpe?g|mp3|mp4|png|ogg|svg|ttf|webm|webp|woff2?)$/.test(req.path) - ) { + if (/\.(gif|jpe?g|mp3|mp4|png|ogg|svg|webm|webp|woff2?)$/.test(req.path)) { // Remember, FileAttachment.findByURLWithFallback() will return the absolute file path // iff it exists on disk. // Using a "fallback" strategy here so that images embedded in live samples From fb903198ad09ee5dd81f236a82f5ae6e48bb6927 Mon Sep 17 00:00:00 2001 From: Claas Augner <495429+caugner@users.noreply.github.com> Date: Wed, 7 Dec 2022 12:13:57 +0100 Subject: [PATCH 12/26] chore(build,filecheck): add param types --- filecheck/checker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filecheck/checker.ts b/filecheck/checker.ts index 0bad88e14f6d..f3dcbc17674f 100644 --- a/filecheck/checker.ts +++ b/filecheck/checker.ts @@ -194,7 +194,7 @@ export async function checkFile( await checkCompression(filePath, options); } -async function checkCompression(filePath, options) { +async function checkCompression(filePath: string, options: CheckerOptions) { const tempdir = temporaryDirectory(); const extension = path.extname(filePath).toLowerCase(); try { From eacc2dac088461c7e10c2ab9759dc2aadbb651fe Mon Sep 17 00:00:00 2001 From: Claas Augner <495429+caugner@users.noreply.github.com> Date: Fri, 9 Dec 2022 15:08:20 +0100 Subject: [PATCH 13/26] fix(filecheck): check extension of ogg files --- filecheck/checker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filecheck/checker.ts b/filecheck/checker.ts index f3dcbc17674f..93154a9f7452 100644 --- a/filecheck/checker.ts +++ b/filecheck/checker.ts @@ -79,7 +79,7 @@ export async function checkFile( } // Ensure that binary files contain what their extension indicates. - if (/\.(mp3|mp4|webm|woff2?)$/i.test(filePath)) { + if (/\.(mp3|mp4|ogg|webm|woff2?)$/i.test(filePath)) { const ext = filePath.split(".").pop(); const type = await fileTypeFromFile(filePath); if (!type) { From 023c6ad32d993eccd64fa2595a77e803fb953f6f Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Fri, 9 Dec 2022 15:35:37 +0100 Subject: [PATCH 14/26] fix(filecheck): reuse sizeBefore --- filecheck/checker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filecheck/checker.ts b/filecheck/checker.ts index 93154a9f7452..065bccb35966 100644 --- a/filecheck/checker.ts +++ b/filecheck/checker.ts @@ -246,7 +246,7 @@ async function checkCompression(filePath: string, options: CheckerOptions) { filePath )} is too large (${formattedBefore} > ${formattedMax}), even after compressing to ${formattedAfter}.` ); - } else if (!options.saveCompression && stat.size > MAX_FILE_SIZE) { + } else if (!options.saveCompression && sizeBefore > MAX_FILE_SIZE) { throw new FixableError( `${getRelativePath( filePath From 357ddb7eee9c11e1ed8cd8a3c7ad0b809f76bb62 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Fri, 16 Dec 2022 11:17:11 +0100 Subject: [PATCH 15/26] chore(file-attachment): only allow woff2, not woff --- build/utils.ts | 2 +- content/file-attachment.ts | 2 +- filecheck/checker.ts | 2 +- server/index.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/utils.ts b/build/utils.ts index e0b0fe0bad2f..0fabc74d189b 100644 --- a/build/utils.ts +++ b/build/utils.ts @@ -191,7 +191,7 @@ export function getAdjacentFileAttachments(documentDirectory: string) { // This needs to match what we do in filecheck/checker.py return ( !dirent.isDirectory() && - /\.(mp3|mp4|png|jpeg|jpg|gif|ogg|svg|webm|webp|woff2?)$/i.test( + /\.(mp3|mp4|png|jpeg|jpg|gif|ogg|svg|webm|webp|woff2)$/i.test( dirent.name ) ); diff --git a/content/file-attachment.ts b/content/file-attachment.ts index 83b0fd309b6a..0cf288c732c3 100644 --- a/content/file-attachment.ts +++ b/content/file-attachment.ts @@ -27,7 +27,7 @@ function isAudio(filePath: string) { } function isFont(filePath: string) { - return /\.(woff2?)$/i.test(filePath); + return /\.(woff2)$/i.test(filePath); } function isVideo(filePath: string) { diff --git a/filecheck/checker.ts b/filecheck/checker.ts index 065bccb35966..23c9b428b413 100644 --- a/filecheck/checker.ts +++ b/filecheck/checker.ts @@ -79,7 +79,7 @@ export async function checkFile( } // Ensure that binary files contain what their extension indicates. - if (/\.(mp3|mp4|ogg|webm|woff2?)$/i.test(filePath)) { + if (/\.(mp3|mp4|ogg|webm|woff2)$/i.test(filePath)) { const ext = filePath.split(".").pop(); const type = await fileTypeFromFile(filePath); if (!type) { diff --git a/server/index.ts b/server/index.ts index 53da336825e3..94adbaca09c4 100644 --- a/server/index.ts +++ b/server/index.ts @@ -294,7 +294,7 @@ app.get("/*", async (req, res, ...args) => { // TODO: Would be nice to have a list of all supported file extensions // in a constants file. - if (/\.(gif|jpe?g|mp3|mp4|png|ogg|svg|webm|webp|woff2?)$/.test(req.path)) { + if (/\.(gif|jpe?g|mp3|mp4|png|ogg|svg|webm|webp|woff2)$/.test(req.path)) { // Remember, FileAttachment.findByURLWithFallback() will return the absolute file path // iff it exists on disk. // Using a "fallback" strategy here so that images embedded in live samples From c024872b84062d5a454ceb98f1b6a45807e3f503 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Thu, 11 May 2023 22:02:16 +0200 Subject: [PATCH 16/26] fix(cloud-function): update asset extensions --- cloud-function/src/app.ts | 2 +- cloud-function/src/utils.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cloud-function/src/app.ts b/cloud-function/src/app.ts index 9197425dd227..6a213b7c9453 100644 --- a/cloud-function/src/app.ts +++ b/cloud-function/src/app.ts @@ -48,7 +48,7 @@ router.get( proxyContent ); router.get( - "/[^/]+/docs/*/*.(png|jpeg|jpg|gif|svg|webp)", + "/[^/]+/docs/*/*.(gif|jpe?g|mp3|mp4|png|ogg|svg|webm|webp|woff2)", requireOrigin(Origin.main, Origin.liveSamples), resolveIndexHTML, proxyContent diff --git a/cloud-function/src/utils.ts b/cloud-function/src/utils.ts index 9479c18168a8..c47a7e3acb4c 100644 --- a/cloud-function/src/utils.ts +++ b/cloud-function/src/utils.ts @@ -45,7 +45,8 @@ export function isLiveSampleURL(url: string) { // These are the only extensions in client/build/*/docs/*. // `find client/build -type f | grep docs | xargs basename | sed 's/.*\.\([^.]*\)$/\1/' | sort | uniq` -const ASSET_REGEXP = /\.(gif|html|jpeg|jpg|json|png|svg|txt|xml)$/i; +const ASSET_REGEXP = + /\.(gif|html|jpe?g|json|mp3|mp4|png|ogg|svg|txt|webm|webp|woff2|xml)$/i; export function isAsset(url: string) { return ASSET_REGEXP.test(url); From 9ebc27f5a64229e8f2a6a2fab2a822881da7ef6b Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Thu, 11 May 2023 22:46:33 +0200 Subject: [PATCH 17/26] refactor(libs/constants): extract extension constants --- build/utils.ts | 12 +++++------- cloud-function/src/app.ts | 4 +++- cloud-function/src/utils.ts | 14 +++++++++++--- content/file-attachment.ts | 22 +++++++++++++++++----- filecheck/checker.ts | 12 +++++++++++- libs/constants/index.d.ts | 10 ++++++++++ libs/constants/index.js | 31 +++++++++++++++++++++++++++++++ server/index.ts | 10 ++++++---- 8 files changed, 94 insertions(+), 21 deletions(-) diff --git a/build/utils.ts b/build/utils.ts index 0fabc74d189b..18766bbafe94 100644 --- a/build/utils.ts +++ b/build/utils.ts @@ -12,7 +12,10 @@ import imageminSvgo from "imagemin-svgo"; import { rgPath } from "@vscode/ripgrep"; import sanitizeFilename from "sanitize-filename"; -import { VALID_MIME_TYPES } from "../libs/constants/index.js"; +import { + ANY_ATTACHMENT_REGEXP, + VALID_MIME_TYPES, +} from "../libs/constants/index.js"; import { FileAttachment } from "../content/index.js"; import { spawnSync } from "node:child_process"; import { BLOG_ROOT } from "../libs/env/index.js"; @@ -189,12 +192,7 @@ export function getAdjacentFileAttachments(documentDirectory: string) { return dirents .filter((dirent) => { // This needs to match what we do in filecheck/checker.py - return ( - !dirent.isDirectory() && - /\.(mp3|mp4|png|jpeg|jpg|gif|ogg|svg|webm|webp|woff2)$/i.test( - dirent.name - ) - ); + return !dirent.isDirectory() && ANY_ATTACHMENT_REGEXP.test(dirent.name); }) .map((dirent) => path.join(documentDirectory, dirent.name)); } diff --git a/cloud-function/src/app.ts b/cloud-function/src/app.ts index 6a213b7c9453..3507fce925da 100644 --- a/cloud-function/src/app.ts +++ b/cloud-function/src/app.ts @@ -1,6 +1,8 @@ import express, { Request, Response } from "express"; import { Router } from "express"; +import { ANY_ATTACHMENT_EXT } from "./internal/constants/index.js"; + import { Origin } from "./env.js"; import { proxyContent } from "./handlers/proxy-content.js"; import { proxyKevel } from "./handlers/proxy-kevel.js"; @@ -48,7 +50,7 @@ router.get( proxyContent ); router.get( - "/[^/]+/docs/*/*.(gif|jpe?g|mp3|mp4|png|ogg|svg|webm|webp|woff2)", + `/[^/]+/docs/*/*.(${ANY_ATTACHMENT_EXT.join("|")})`, requireOrigin(Origin.main, Origin.liveSamples), resolveIndexHTML, proxyContent diff --git a/cloud-function/src/utils.ts b/cloud-function/src/utils.ts index c47a7e3acb4c..34f0060d6608 100644 --- a/cloud-function/src/utils.ts +++ b/cloud-function/src/utils.ts @@ -1,5 +1,10 @@ import { Request, Response } from "express"; +import { + ANY_ATTACHMENT_EXT, + createRegExpFromExtensions, +} from "./internal/constants/index.js"; + import { DEFAULT_COUNTRY } from "./constants.js"; export function getRequestCountry(req: Request): string { @@ -45,9 +50,12 @@ export function isLiveSampleURL(url: string) { // These are the only extensions in client/build/*/docs/*. // `find client/build -type f | grep docs | xargs basename | sed 's/.*\.\([^.]*\)$/\1/' | sort | uniq` -const ASSET_REGEXP = - /\.(gif|html|jpe?g|json|mp3|mp4|png|ogg|svg|txt|webm|webp|woff2|xml)$/i; +const TEXT_EXT = ["html", "json", "svg", "txt", "xml"]; +const ANY_ATTACHMENT_REGEXP = createRegExpFromExtensions( + ...ANY_ATTACHMENT_EXT, + ...TEXT_EXT +); export function isAsset(url: string) { - return ASSET_REGEXP.test(url); + return ANY_ATTACHMENT_REGEXP.test(url); } diff --git a/content/file-attachment.ts b/content/file-attachment.ts index 0cf288c732c3..152c66298639 100644 --- a/content/file-attachment.ts +++ b/content/file-attachment.ts @@ -5,7 +5,14 @@ import { readChunkSync } from "read-chunk"; import imageType from "image-type"; import isSvg from "is-svg"; -import { DEFAULT_LOCALE } from "../libs/constants/index.js"; +import { + ANY_IMAGE_EXT, + AUDIO_EXT, + DEFAULT_LOCALE, + FONT_EXT, + VIDEO_EXT, + createRegExpFromExtensions, +} from "../libs/constants/index.js"; import { ROOTS } from "../libs/env/index.js"; import { memoize, slugToFolder } from "./utils.js"; @@ -22,20 +29,25 @@ function isFileAttachment(filePath: string) { ); } +const AUDIO_FILE_REGEXP = createRegExpFromExtensions(...AUDIO_EXT); +const FONT_FILE_REGEXP = createRegExpFromExtensions(...FONT_EXT); +const VIDEO_FILE_REGEXP = createRegExpFromExtensions(...VIDEO_EXT); +const IMAGE_FILE_REGEXP = createRegExpFromExtensions(...ANY_IMAGE_EXT); + function isAudio(filePath: string) { - return /\.(mp3|ogg)$/i.test(filePath); + return AUDIO_FILE_REGEXP.test(filePath); } function isFont(filePath: string) { - return /\.(woff2)$/i.test(filePath); + return FONT_FILE_REGEXP.test(filePath); } function isVideo(filePath: string) { - return /\.(mp4|webm)$/i.test(filePath); + return VIDEO_FILE_REGEXP.test(filePath); } function isImage(filePath: string) { - if (!/\.(gif|jpe?g|png|svg)$/i.test(filePath)) { + if (!IMAGE_FILE_REGEXP.test(filePath)) { return false; } diff --git a/filecheck/checker.ts b/filecheck/checker.ts index 23c9b428b413..150a68135b18 100644 --- a/filecheck/checker.ts +++ b/filecheck/checker.ts @@ -20,10 +20,20 @@ import { MAX_FILE_SIZE } from "../libs/env/index.js"; import { VALID_MIME_TYPES, MAX_COMPRESSION_DIFFERENCE_PERCENTAGE, + createRegExpFromExtensions, + AUDIO_EXT, + VIDEO_EXT, + FONT_EXT, } from "../libs/constants/index.js"; const { default: imageminPngquant } = imageminPngquantPkg; +const BINARY_FILE_REGEXP = createRegExpFromExtensions( + ...AUDIO_EXT, + ...VIDEO_EXT, + ...FONT_EXT +); + function formatSize(bytes: number): string { if (bytes > 1024 * 1024) { return `${(bytes / 1024.0 / 1024.0).toFixed(1)}MB`; @@ -79,7 +89,7 @@ export async function checkFile( } // Ensure that binary files contain what their extension indicates. - if (/\.(mp3|mp4|ogg|webm|woff2)$/i.test(filePath)) { + if (BINARY_FILE_REGEXP.test(filePath)) { const ext = filePath.split(".").pop(); const type = await fileTypeFromFile(filePath); if (!type) { diff --git a/libs/constants/index.d.ts b/libs/constants/index.d.ts index 296aa927d7b6..04e2deaad14b 100644 --- a/libs/constants/index.d.ts +++ b/libs/constants/index.d.ts @@ -6,6 +6,16 @@ export const LOCALE_ALIASES: Map; export const PREFERRED_LOCALE_COOKIE_NAME: string; export const CSP_SCRIPT_SRC_VALUES: string[]; export const CSP_VALUE: string; +export const AUDIO_EXT: string[]; +export const FONT_EXT: string[]; +export const BINARY_IMAGE_EXT: string[]; +export const ANY_IMAGE_EXT: string[]; +export const VIDEO_EXT: string[]; +export const ANY_ATTACHMENT_EXT: string[]; +export const BINARY_ATTACHMENT_EXT: string[]; +export const createRegExpFromExtensions: (...extensions: string[]) => RegExp; +export const ANY_ATTACHMENT_REGEXP: RegExp; +export const BINARY_ATTACHMENT_REGEXP: RegExp; export const FLAW_LEVELS: Readonly>; export const VALID_FLAW_CHECKS: Set; export const MDN_PLUS_TITLE: string; diff --git a/libs/constants/index.js b/libs/constants/index.js index 94d18d27ccc1..53fd0e1d5124 100644 --- a/libs/constants/index.js +++ b/libs/constants/index.js @@ -153,6 +153,37 @@ export const cspToString = (csp) => export const CSP_VALUE = cspToString(CSP_DIRECTIVES); +export const AUDIO_EXT = ["mp3", "ogg"]; +export const FONT_EXT = ["woff2"]; +export const BINARY_IMAGE_EXT = ["gif", "jpeg", "jpg", "png", "webp"]; +export const ANY_IMAGE_EXT = ["svg", ...BINARY_IMAGE_EXT]; +export const VIDEO_EXT = ["mp4", "webm"]; + +export const BINARY_ATTACHMENT_EXT = [ + ...AUDIO_EXT, + ...FONT_EXT, + ...BINARY_IMAGE_EXT, + ...VIDEO_EXT, +].sort(); + +export const ANY_ATTACHMENT_EXT = [ + ...AUDIO_EXT, + ...FONT_EXT, + ...ANY_IMAGE_EXT, + ...VIDEO_EXT, +].sort(); + +export function createRegExpFromExtensions(...extensions) { + return new RegExp(`\\.(${extensions.join("|")})$`, "i"); +} + +export const ANY_ATTACHMENT_REGEXP = createRegExpFromExtensions( + ...ANY_ATTACHMENT_EXT +); +export const BINARY_ATTACHMENT_REGEXP = createRegExpFromExtensions( + ...BINARY_ATTACHMENT_EXT +); + // ----- // build // ----- diff --git a/server/index.ts b/server/index.ts index 94adbaca09c4..463585520b00 100644 --- a/server/index.ts +++ b/server/index.ts @@ -18,7 +18,11 @@ import { } from "../build/index.js"; import { findTranslations } from "../content/translations.js"; import { Document, Redirect, FileAttachment } from "../content/index.js"; -import { CSP_VALUE, DEFAULT_LOCALE } from "../libs/constants/index.js"; +import { + ANY_ATTACHMENT_REGEXP, + CSP_VALUE, + DEFAULT_LOCALE, +} from "../libs/constants/index.js"; import { STATIC_ROOT, PROXY_HOSTNAME, @@ -292,9 +296,7 @@ app.get("/*", async (req, res, ...args) => { .sendFile(path.join(STATIC_ROOT, "en-us", "_spas", "404.html")); } - // TODO: Would be nice to have a list of all supported file extensions - // in a constants file. - if (/\.(gif|jpe?g|mp3|mp4|png|ogg|svg|webm|webp|woff2)$/.test(req.path)) { + if (ANY_ATTACHMENT_REGEXP.test(req.path)) { // Remember, FileAttachment.findByURLWithFallback() will return the absolute file path // iff it exists on disk. // Using a "fallback" strategy here so that images embedded in live samples From 33ec69fa09df1e29705433d753993dc307b335cc Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Thu, 11 May 2023 22:55:23 +0200 Subject: [PATCH 18/26] test(libs/constants): add test cases + new test:lib script --- .github/workflows/testing.yml | 3 ++ libs/constants/index.test.ts | 62 +++++++++++++++++++++++++++++++++++ package.json | 3 +- 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 libs/constants/index.test.ts diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 214da10534dc..33cad9688012 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -69,6 +69,9 @@ jobs: - name: Unit testing client run: yarn test:client + - name: Unit testing libs + run: yarn test:libs + - name: Build and start server id: server env: diff --git a/libs/constants/index.test.ts b/libs/constants/index.test.ts new file mode 100644 index 000000000000..f3d309e5d3e7 --- /dev/null +++ b/libs/constants/index.test.ts @@ -0,0 +1,62 @@ +import { + createRegExpFromExtensions, + ANY_ATTACHMENT_REGEXP, + BINARY_ATTACHMENT_REGEXP, +} from "./index.js"; + +describe("createRegExpFromExt", () => { + const regexp = createRegExpFromExtensions("foo"); + + it("accepts the extension", () => { + expect(regexp.test("test.foo")).toEqual(true); + }); + + it("accepts uppercase", () => { + expect(regexp.test("test.FOO")).toEqual(true); + }); + + it("rejects intermediate extensions", () => { + expect(regexp.test("test.foo.bar")).toEqual(false); + }); + + it("rejects other extensions", () => { + expect(regexp.test("test.bar")).toEqual(false); + }); + + it("rejects extensions starting with it", () => { + expect(regexp.test("test.foob")).toEqual(false); + }); + + it("rejects extensions ending with it", () => { + expect(regexp.test("test.afoo")).toEqual(false); + }); +}); + +describe("ANY_ATTACHMENT_REGEXP", () => { + const regexp = ANY_ATTACHMENT_REGEXP; + it("accepts audio files", () => { + expect(regexp.test("audio.mp3")).toEqual(true); + }); + + it("accepts video files", () => { + expect(regexp.test("video.mp4")).toEqual(true); + }); + + it("accepts font files", () => { + expect(regexp.test("diagram.svg")).toEqual(true); + }); + + ["index.html", "index.json", "index.md", "contributors.txt"].forEach( + (filename) => + it(`rejects ${filename}`, () => { + expect(regexp.test(filename)).toEqual(false); + }) + ); +}); + +describe("BINARY_ATTACHMENT_REGEXP", () => { + const regexp = BINARY_ATTACHMENT_REGEXP; + it("rejects svg files", () => { + expect(regexp.test("diagram.svg")).toEqual(false); + }); +}); diff --git a/package.json b/package.json index e653c15e6cfa..562874c46fe9 100644 --- a/package.json +++ b/package.json @@ -42,12 +42,13 @@ "start:static-server": "ts-node server/static.ts", "style-dictionary": "style-dictionary build -c sd-config.js", "stylelint": "stylelint \"**/*.scss\"", - "test": "yarn prettier-check && yarn test:client && yarn test:kumascript && yarn test:content && yarn test:testing", + "test": "yarn prettier-check && yarn test:client && yarn test:kumascript && yarn test:libs && yarn test:content && yarn test:testing", "test:client": "cd client && tsc --noEmit && cross-env NODE_ENV=test BABEL_ENV=test PUBLIC_URL='' node scripts/test.js --env=jsdom", "test:content": "yarn jest --rootDir content", "test:developing": "cross-env CONTENT_ROOT=mdn/content/files TESTING_DEVELOPING=true playwright test developing", "test:headless": "playwright test headless", "test:kumascript": "yarn jest --rootDir kumascript --env=node", + "test:libs": "yarn jest --rootDir libs --env=node", "test:prepare": "yarn build:prepare && yarn build && yarn start:static-server", "test:testing": "yarn jest --rootDir testing", "tool": "ts-node tool/cli.ts", From c24bd17818cb7a2e0593c97883e5438372ee8a6d Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Thu, 11 May 2023 23:05:11 +0200 Subject: [PATCH 19/26] chore(filecheck/checker): make binary file check clearer --- filecheck/checker.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/filecheck/checker.ts b/filecheck/checker.ts index 150a68135b18..742bf51baff7 100644 --- a/filecheck/checker.ts +++ b/filecheck/checker.ts @@ -28,7 +28,7 @@ import { const { default: imageminPngquant } = imageminPngquantPkg; -const BINARY_FILE_REGEXP = createRegExpFromExtensions( +const BINARY_NON_IMAGE_FILE_REGEXP = createRegExpFromExtensions( ...AUDIO_EXT, ...VIDEO_EXT, ...FONT_EXT @@ -89,7 +89,8 @@ export async function checkFile( } // Ensure that binary files contain what their extension indicates. - if (BINARY_FILE_REGEXP.test(filePath)) { + // Exclude images, as they're checked separately in checkCompression(). + if (BINARY_NON_IMAGE_FILE_REGEXP.test(filePath)) { const ext = filePath.split(".").pop(); const type = await fileTypeFromFile(filePath); if (!type) { From 037ac8087013fffceda53f9b9c92d86eb24a2166 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Thu, 11 May 2023 23:13:01 +0200 Subject: [PATCH 20/26] feat(docs): allow avif video assets --- libs/constants/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/constants/index.js b/libs/constants/index.js index 53fd0e1d5124..e166e7d2160d 100644 --- a/libs/constants/index.js +++ b/libs/constants/index.js @@ -157,7 +157,7 @@ export const AUDIO_EXT = ["mp3", "ogg"]; export const FONT_EXT = ["woff2"]; export const BINARY_IMAGE_EXT = ["gif", "jpeg", "jpg", "png", "webp"]; export const ANY_IMAGE_EXT = ["svg", ...BINARY_IMAGE_EXT]; -export const VIDEO_EXT = ["mp4", "webm"]; +export const VIDEO_EXT = ["avif", "mp4", "webm"]; export const BINARY_ATTACHMENT_EXT = [ ...AUDIO_EXT, @@ -243,6 +243,7 @@ export const VALID_MIME_TYPES = new Set([ "image/jpeg", // this is what you get for .jpeg *and* .jpg file extensions "image/gif", "image/webp", + "video/avif", "video/mp4", "video/ogg", "video/webm", From ae818dbe0d550066985e17e05f923c04fad24f4e Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Thu, 11 May 2023 23:21:01 +0200 Subject: [PATCH 21/26] chore(build): update some comments --- build/flaws/broken-links.ts | 2 +- build/utils.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build/flaws/broken-links.ts b/build/flaws/broken-links.ts index 9374a60e4e84..f95ed9a60963 100644 --- a/build/flaws/broken-links.ts +++ b/build/flaws/broken-links.ts @@ -277,7 +277,7 @@ export function getBrokenLinksFlaws( const absoluteURL = new URL(href, "http://www.example.com"); const found = Document.findByURL(hrefNormalized); if (!found) { - // Before we give up, check if it's an image. + // Before we give up, check if it's an attachment. if (!FileAttachment.findByURLWithFallback(hrefNormalized)) { // Even if it's a redirect, it's still a flaw, but it'll be nice to // know what it *should* be. diff --git a/build/utils.ts b/build/utils.ts index 18766bbafe94..f1d6151bfc53 100644 --- a/build/utils.ts +++ b/build/utils.ts @@ -249,9 +249,9 @@ export function postLocalFileLinks($, doc) { const href = element.attribs.href; // This test is merely here to quickly bail if there's no hope to find the - // image as a local file link. There are a LOT of hyperlinks throughout - // the content and this simple if statement means we can skip 99% of the - // links, so it's presumed to be worth it. + // file attachment as a local file link. There are a LOT of hyperlinks + // throughout the content and this simple if statement means we can skip 99% + // of the links, so it's presumed to be worth it. if ( !href || /^(\/|\.\.|http|#|mailto:|about:|ftp:|news:|irc:|ftp:)/i.test(href) @@ -259,11 +259,11 @@ export function postLocalFileLinks($, doc) { return; } // There are a lot of links that don't match. E.g. `` - // So we'll look-up a lot "false positives" that are not images. + // So we'll look-up a lot "false positives" that are not file attachments. // Thankfully, this lookup is fast. const url = `${doc.mdn_url}/${href}`; - const image = FileAttachment.findByURLWithFallback(url); - if (image) { + const fileAttachment = FileAttachment.findByURLWithFallback(url); + if (fileAttachment) { $(element).attr("href", url); } }); From c22d47d6f400026e987a448f99568065f6b47837 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Fri, 12 May 2023 15:59:13 +0200 Subject: [PATCH 22/26] Revert "feat(docs): allow avif video assets" This reverts commit ee1f286f16781d71c72b844c9bb3fab503fc6ff6. --- libs/constants/index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/constants/index.js b/libs/constants/index.js index e166e7d2160d..53fd0e1d5124 100644 --- a/libs/constants/index.js +++ b/libs/constants/index.js @@ -157,7 +157,7 @@ export const AUDIO_EXT = ["mp3", "ogg"]; export const FONT_EXT = ["woff2"]; export const BINARY_IMAGE_EXT = ["gif", "jpeg", "jpg", "png", "webp"]; export const ANY_IMAGE_EXT = ["svg", ...BINARY_IMAGE_EXT]; -export const VIDEO_EXT = ["avif", "mp4", "webm"]; +export const VIDEO_EXT = ["mp4", "webm"]; export const BINARY_ATTACHMENT_EXT = [ ...AUDIO_EXT, @@ -243,7 +243,6 @@ export const VALID_MIME_TYPES = new Set([ "image/jpeg", // this is what you get for .jpeg *and* .jpg file extensions "image/gif", "image/webp", - "video/avif", "video/mp4", "video/ogg", "video/webm", From 19944c7dd6a318275b0e62fc06c748c078e05d9e Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Mon, 29 May 2023 10:06:21 +0200 Subject: [PATCH 23/26] fix(webpack): declare audio/video/font files as modules --- client/src/react-app.d.ts | 25 +++++++++++++++++++++++++ server/react-app.d.ts | 25 +++++++++++++++++++++++++ ssr/react-app.d.ts | 25 +++++++++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/client/src/react-app.d.ts b/client/src/react-app.d.ts index 4a3ff36d684d..6084dd416356 100644 --- a/client/src/react-app.d.ts +++ b/client/src/react-app.d.ts @@ -34,16 +34,41 @@ declare module "*.jpeg" { export default src; } +declare module "*.mp3" { + const src: string; + export default src; +} + +declare module "*.mp4" { + const src: string; + export default src; +} + +declare module "*.ogg" { + const src: string; + export default src; +} + declare module "*.png" { const src: string; export default src; } +declare module "*.webm" { + const src: string; + export default src; +} + declare module "*.webp" { const src: string; export default src; } +declare module "*.woff2" { + const src: string; + export default src; +} + declare module "*.svg" { import * as React from "react"; diff --git a/server/react-app.d.ts b/server/react-app.d.ts index bf4ab9180050..ba232bc32748 100644 --- a/server/react-app.d.ts +++ b/server/react-app.d.ts @@ -30,16 +30,41 @@ declare module "*.jpeg" { export default src; } +declare module "*.mp3" { + const src: string; + export default src; +} + +declare module "*.mp4" { + const src: string; + export default src; +} + +declare module "*.ogg" { + const src: string; + export default src; +} + declare module "*.png" { const src: string; export default src; } +declare module "*.webm" { + const src: string; + export default src; +} + declare module "*.webp" { const src: string; export default src; } +declare module "*.woff2" { + const src: string; + export default src; +} + declare module "*.svg" { import * as React from "react"; diff --git a/ssr/react-app.d.ts b/ssr/react-app.d.ts index bf4ab9180050..ba232bc32748 100644 --- a/ssr/react-app.d.ts +++ b/ssr/react-app.d.ts @@ -30,16 +30,41 @@ declare module "*.jpeg" { export default src; } +declare module "*.mp3" { + const src: string; + export default src; +} + +declare module "*.mp4" { + const src: string; + export default src; +} + +declare module "*.ogg" { + const src: string; + export default src; +} + declare module "*.png" { const src: string; export default src; } +declare module "*.webm" { + const src: string; + export default src; +} + declare module "*.webp" { const src: string; export default src; } +declare module "*.woff2" { + const src: string; + export default src; +} + declare module "*.svg" { import * as React from "react"; From 04f2d165db359e57bcb04a1bd0845bcedf0ad458 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Fri, 2 Jun 2023 15:33:11 +0200 Subject: [PATCH 24/26] fix(client): update proxied extensions for dev server --- client/src/setupProxy.js | 4 ++-- libs/constants/index.js | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/client/src/setupProxy.js b/client/src/setupProxy.js index dba0bbd277c7..02d3a0a312f0 100644 --- a/client/src/setupProxy.js +++ b/client/src/setupProxy.js @@ -16,8 +16,8 @@ function config(app) { app.use("/_+(flaws|translations|open|document)", proxy); // E.g. search-index.json or index.json app.use("**/*.json", proxy); - // This has to match what we do in server/index.js in the catchall handler - app.use("**/*.(png|webp|gif|jpe?g|svg)", proxy); + // This must match extensions in libs/constant/index.js:156-161, or it won't work on the dev server. + app.use(`**/*.(gif|jpeg|jpg|mp3|mp4|ogg|png|svg|webm|webp|woff2)`, proxy); // All those root-level images like /favicon-48x48.png app.use("/*.(png|webp|gif|jpe?g|svg)", proxy); } diff --git a/libs/constants/index.js b/libs/constants/index.js index 53fd0e1d5124..b87825193d75 100644 --- a/libs/constants/index.js +++ b/libs/constants/index.js @@ -153,6 +153,7 @@ export const cspToString = (csp) => export const CSP_VALUE = cspToString(CSP_DIRECTIVES); +// Always update client/src/setupProxy.js:20 when adding/removing extensions! export const AUDIO_EXT = ["mp3", "ogg"]; export const FONT_EXT = ["woff2"]; export const BINARY_IMAGE_EXT = ["gif", "jpeg", "jpg", "png", "webp"]; From cddc538d452c12bdcc5da0b0ad4c464d08f62630 Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Fri, 2 Jun 2023 15:44:52 +0200 Subject: [PATCH 25/26] chore: update comments --- client/src/setupProxy.js | 2 +- libs/constants/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/src/setupProxy.js b/client/src/setupProxy.js index 02d3a0a312f0..8ab9ed5a2c2b 100644 --- a/client/src/setupProxy.js +++ b/client/src/setupProxy.js @@ -16,7 +16,7 @@ function config(app) { app.use("/_+(flaws|translations|open|document)", proxy); // E.g. search-index.json or index.json app.use("**/*.json", proxy); - // This must match extensions in libs/constant/index.js:156-161, or it won't work on the dev server. + // Always update libs/constant/index.js when adding/removing extensions! app.use(`**/*.(gif|jpeg|jpg|mp3|mp4|ogg|png|svg|webm|webp|woff2)`, proxy); // All those root-level images like /favicon-48x48.png app.use("/*.(png|webp|gif|jpe?g|svg)", proxy); diff --git a/libs/constants/index.js b/libs/constants/index.js index b87825193d75..bc5ae47321d4 100644 --- a/libs/constants/index.js +++ b/libs/constants/index.js @@ -153,7 +153,7 @@ export const cspToString = (csp) => export const CSP_VALUE = cspToString(CSP_DIRECTIVES); -// Always update client/src/setupProxy.js:20 when adding/removing extensions! +// Always update client/src/setupProxy.js when adding/removing extensions, or it won't work on the dev server! export const AUDIO_EXT = ["mp3", "ogg"]; export const FONT_EXT = ["woff2"]; export const BINARY_IMAGE_EXT = ["gif", "jpeg", "jpg", "png", "webp"]; From 7e3326aabed1a420e17c0870deeceb6a9c26b24f Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Sat, 3 Jun 2023 00:10:11 +0200 Subject: [PATCH 26/26] chore(libs/constants): remove font/woff --- libs/constants/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/constants/index.js b/libs/constants/index.js index bc5ae47321d4..d670023c3ee2 100644 --- a/libs/constants/index.js +++ b/libs/constants/index.js @@ -238,7 +238,6 @@ export const VALID_MIME_TYPES = new Set([ "audio/mpeg", "audio/ogg", "audio/webm", - "font/woff", "font/woff2", "image/png", "image/jpeg", // this is what you get for .jpeg *and* .jpg file extensions