From 4f9d8eaa3eb5a7930731cb5ee3f3b8cc6343dc65 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 22 Jul 2024 17:19:32 -0700 Subject: [PATCH 1/4] wip: all: highlight with shiki in production --- assets/scss/common/_custom.scss | 18 +++ config/_default/markup.toml | 12 +- .../_default/_markup/render-codeblock.html | 19 +++ package-lock.json | 126 +++++++++++++++++- package.json | 9 +- scripts/highlight.mjs | 70 ++++++++++ 6 files changed, 240 insertions(+), 14 deletions(-) create mode 100644 layouts/_default/_markup/render-codeblock.html create mode 100644 scripts/highlight.mjs diff --git a/assets/scss/common/_custom.scss b/assets/scss/common/_custom.scss index 3b94419..7831d93 100644 --- a/assets/scss/common/_custom.scss +++ b/assets/scss/common/_custom.scss @@ -18,3 +18,21 @@ a.broken { .quick-links-container { max-width: 900px; } + +pre.shiki { + padding: 1.25rem 1.25rem; + border: 1px solid color-mix(in srgb, var(--sl-color-gray-5), transparent 25%); +} + +// Make Shiki syntax highlighting theme-aware. +html[data-bs-theme="dark"] { + .shiki, + .shiki span { + color: var(--shiki-dark) !important; + background-color: var(--shiki-dark-bg) !important; + /* Optional, if you also want font styles */ + font-style: var(--shiki-dark-font-style) !important; + font-weight: var(--shiki-dark-font-weight) !important; + text-decoration: var(--shiki-dark-text-decoration) !important; + } +} diff --git a/config/_default/markup.toml b/config/_default/markup.toml index 8205ba8..e257c7b 100644 --- a/config/_default/markup.toml +++ b/config/_default/markup.toml @@ -20,19 +20,9 @@ defaultMarkdownHandler = "goldmark" enableDefault = true [highlight] - anchorLineNos = false - codeFences = true - guessSyntax = false - hl_Lines = '' hl_inline = false - lineAnchors = '' - lineNoStart = 1 - lineNos = false - lineNumbersInTable = false + codeFences = true noClasses = false - noHl = false - style = 'monokai' - tabWidth = 2 [tableOfContents] endLevel = 3 diff --git a/layouts/_default/_markup/render-codeblock.html b/layouts/_default/_markup/render-codeblock.html new file mode 100644 index 0000000..72eda41 --- /dev/null +++ b/layouts/_default/_markup/render-codeblock.html @@ -0,0 +1,19 @@ +{{- if hugo.IsProduction }} + {{- /* + In production, scripts/highlight.mjs post-processes all codeblocks with Shiki, + so do not transform the codeblock here. + */ -}} + 
{{ .Inner }}
+{{- else }} + {{- /* + In development, fall back on the faster default highlighting built in to Doks. + See https://github.com/gethyas/doks-core/blob/main/layouts/_default/_markup/render-codeblock.html. + */}} + {{- $result := transform.HighlightCodeBlock . -}} +
+
+ {{ $result.Wrapped }} +
+ +
+{{- end -}} diff --git a/package-lock.json b/package-lock.json index 66269b2..8906e55 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,11 @@ "@hyas/inline-svg": "^1.1.0", "@hyas/seo": "^2.3.0", "@tabler/icons": "^3.2.0", - "gethyas": "^2.4.2" + "dom-serializer": "^2.0.0", + "domutils": "^3.1.0", + "gethyas": "^2.4.2", + "htmlparser2": "^9.1.0", + "shiki": "^1.11.0" }, "devDependencies": { "vite": "^5.2.10" @@ -2480,6 +2484,15 @@ "win32" ] }, + "node_modules/@shikijs/core": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-1.11.0.tgz", + "integrity": "sha512-VbEhDAhT/2ozO0TPr5/ZQBO/NWLqtk4ZiBf6NplYpF38mKjNfMMied5fNEfIfYfN+cdKvhDB4VMcKvG/g9c3zg==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.4" + } + }, "node_modules/@sindresorhus/merge-streams": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-2.3.0.tgz", @@ -2506,6 +2519,21 @@ "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", "dev": true }, + "node_modules/@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/unist": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.2.tgz", + "integrity": "sha512-dqId9J8K/vGi5Zr7oo212BGii5m3q5Hxlkwy3WpYuKPklmBEvsbMYYyLxAQpSffdLl/gdW0XUpKWFvYmyoWCoQ==", + "license": "MIT" + }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -2884,6 +2912,61 @@ "node": ">= 0.6.0" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -2899,6 +2982,18 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/esbuild": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz", @@ -3339,6 +3434,25 @@ "node": ">= 0.4" } }, + "node_modules/htmlparser2": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", + "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "entities": "^4.5.0" + } + }, "node_modules/ignore": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", @@ -4366,6 +4480,16 @@ "node": ">=8" } }, + "node_modules/shiki": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-1.11.0.tgz", + "integrity": "sha512-NqH/O1zRHvnuk/WfSL6b7+DtI7/kkMMSQGlZhm9DyzSU+SoIHhaw/fBZMr+zp9R8KjdIzkk3JKSC6hORuGDyng==", + "license": "MIT", + "dependencies": { + "@shikijs/core": "1.11.0", + "@types/hast": "^3.0.4" + } + }, "node_modules/signal-exit": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", diff --git a/package.json b/package.json index 78559e0..b6c903a 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "create": "hugo new", "dev": "hugo server --disableFastRender --noHTTPCache", "format": "prettier **/** -w -c", - "build": "hugo --minify --gc", + "build": "hugo --minify --gc && npm run highlight", + "highlight": "node scripts/highlight.mjs", "preview": "vite preview --outDir public" }, "dependencies": { @@ -18,7 +19,11 @@ "@hyas/inline-svg": "^1.1.0", "@hyas/seo": "^2.3.0", "@tabler/icons": "^3.2.0", - "gethyas": "^2.4.2" + "dom-serializer": "^2.0.0", + "domutils": "^3.1.0", + "gethyas": "^2.4.2", + "htmlparser2": "^9.1.0", + "shiki": "^1.11.0" }, "devDependencies": { "vite": "^5.2.10" diff --git a/scripts/highlight.mjs b/scripts/highlight.mjs new file mode 100644 index 0000000..630342c --- /dev/null +++ b/scripts/highlight.mjs @@ -0,0 +1,70 @@ +// @ts-check +import { createHighlighter, bundledLanguages } from 'shiki'; + +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { readdir, readFile, writeFile } from 'node:fs/promises'; + +import { parseDocument } from 'htmlparser2'; +import { findAll, textContent, replaceElement } from 'domutils'; +import render from 'dom-serializer'; + +highlightAll({ lightTheme: 'github-light-default', darkTheme: 'github-dark-default' }); + +async function highlightAll({ lightTheme, darkTheme }) { + const start = Date.now(); + + const files = await listBuiltContentFiles(); + const highlighter = await createHighlighter({ + themes: [lightTheme, darkTheme], + langs: Object.keys(bundledLanguages), + }); + await Promise.all(files.map((filepath) => highlightFile(highlighter, filepath, { lightTheme, darkTheme }))); + + console.log(`Highlighted ${files.length} files in ${Math.round(Date.now() - start)} ms`); +} + +// Return the paths of all index.html files under the public/docs and public/learn directories. +async function listBuiltContentFiles() { + const publicDir = join(dirname(fileURLToPath(import.meta.url)), '../public'); + + const docsDir = join(publicDir, 'docs'); + const docsIndexFiles = (await readdir(docsDir, { recursive: true, encoding: 'utf-8' })) + .filter((filename) => filename.endsWith('index.html')) + .map((filename) => join(docsDir, filename)); + + const learnDir = join(publicDir, 'learn'); + const learnIndexFiles = (await readdir(learnDir, { recursive: true, encoding: 'utf-8' })) + .filter((filename) => filename.endsWith('index.html')) + .map((filename) => join(learnDir, filename)); + + return [...docsIndexFiles, ...learnIndexFiles]; +} + +async function highlightFile(highlighter, filepath, { lightTheme, darkTheme }) { + const contents = await readFile(filepath, { encoding: 'utf-8' }); + await writeFile(filepath, highlightHtml(highlighter, contents, { lightTheme, darkTheme })); +} + +const codeLanguageRe = /^/; + +function highlightHtml(highlighter, htmlContent, { lightTheme, darkTheme }) { + const doc = parseDocument(htmlContent); + for (const preNode of findAll((e) => e.name === 'pre', doc.children)) { + const codeNode = preNode.childNodes[0]; + if (!codeNode) continue; + + const lang = codeLanguageRe.exec(render(codeNode))?.[1] ?? 'text'; + + const code = textContent(codeNode); + const highlighted = highlighter.codeToHtml(code, { + lang, + themes: { light: lightTheme, dark: darkTheme }, + }); + + const highlightedPreNode = parseDocument(highlighted).childNodes[0]; + replaceElement(preNode, highlightedPreNode); + } + + return render(doc); +} From 35cfb1b532a212bed27404d2b6cd33c569ace1de Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 22 Jul 2024 17:41:31 -0700 Subject: [PATCH 2/4] scripts/highlight: simplify code language regex --- scripts/highlight.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/highlight.mjs b/scripts/highlight.mjs index 630342c..ae821ff 100644 --- a/scripts/highlight.mjs +++ b/scripts/highlight.mjs @@ -46,7 +46,7 @@ async function highlightFile(highlighter, filepath, { lightTheme, darkTheme }) { await writeFile(filepath, highlightHtml(highlighter, contents, { lightTheme, darkTheme })); } -const codeLanguageRe = /^/; +const codeLanguageRe = //; function highlightHtml(highlighter, htmlContent, { lightTheme, darkTheme }) { const doc = parseDocument(htmlContent); From 901a0095eaca35529bc6538a4ee910d8092a3452 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 22 Jul 2024 17:45:02 -0700 Subject: [PATCH 3/4] scripts/highlight: rename entrypoint to main() --- scripts/highlight.mjs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/highlight.mjs b/scripts/highlight.mjs index ae821ff..92df13b 100644 --- a/scripts/highlight.mjs +++ b/scripts/highlight.mjs @@ -9,17 +9,22 @@ import { parseDocument } from 'htmlparser2'; import { findAll, textContent, replaceElement } from 'domutils'; import render from 'dom-serializer'; -highlightAll({ lightTheme: 'github-light-default', darkTheme: 'github-dark-default' }); +main(); + +async function main() { + const LIGHT_THEME = 'github-light-default'; + const DARK_THEME = 'github-dark-default'; -async function highlightAll({ lightTheme, darkTheme }) { const start = Date.now(); const files = await listBuiltContentFiles(); const highlighter = await createHighlighter({ - themes: [lightTheme, darkTheme], + themes: [LIGHT_THEME, DARK_THEME], langs: Object.keys(bundledLanguages), }); - await Promise.all(files.map((filepath) => highlightFile(highlighter, filepath, { lightTheme, darkTheme }))); + await Promise.all( + files.map((filepath) => highlightFile(highlighter, filepath, { lightTheme: LIGHT_THEME, darkTheme: DARK_THEME })), + ); console.log(`Highlighted ${files.length} files in ${Math.round(Date.now() - start)} ms`); } From 962f42c5c59c0cc761eb0006060ca23b8e30f119 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 22 Jul 2024 19:01:08 -0700 Subject: [PATCH 4/4] highlight: add back copy button --- assets/scss/common/_custom.scss | 102 ++++++++++++++++++ .../_default/_markup/render-codeblock.html | 4 +- 2 files changed, 105 insertions(+), 1 deletion(-) diff --git a/assets/scss/common/_custom.scss b/assets/scss/common/_custom.scss index 7831d93..3b68f15 100644 --- a/assets/scss/common/_custom.scss +++ b/assets/scss/common/_custom.scss @@ -36,3 +36,105 @@ html[data-bs-theme="dark"] { text-decoration: var(--shiki-dark-text-decoration) !important; } } + +// Add a copy button to codeblocks. Adapted from +// https://github.com/gethyas/doks-core/blob/main/assets/scss/components/_expressive-code.scss. +.highlight .copy { + display: flex; + gap: 0.25rem; + flex-direction: row; + position: absolute; + inset-block-start: 0.75rem; + inset-inline-end: 0.75rem; + direction: ltr; + unicode-bidi: isolate; +} + +.highlight .copy button { + position: relative; + align-self: flex-end; + margin: 0; + padding: 0; + border: none; + border-radius: 0.2rem; + z-index: 1; + cursor: pointer; + transition-property: opacity, background, border-color; + transition-duration: 0.2s; + transition-timing-function: cubic-bezier(0.25, 0.46, 0.45, 0.94); + width: 2.5rem; + height: 2.5rem; + opacity: 0.75; +} + +.highlight .copy button div { + position: absolute; + inset: 0; + border-radius: inherit; + background: var(--ec-frm-inlBtnBg); + opacity: var(--ec-frm-inlBtnBgIdleOpa); + transition-property: inherit; + transition-duration: inherit; + transition-timing-function: inherit; +} + +.highlight .copy button::before { + content: ""; + position: absolute; + pointer-events: none; + inset: 0; + border-radius: inherit; + border: var(--ec-brdWd) solid var(--ec-frm-inlBtnBrd); + opacity: var(--ec-frm-inlBtnBrdOpa); +} + +.highlight .copy button::after { + content: ""; + position: absolute; + pointer-events: none; + inset: 0; + background-color: var(--ec-frm-inlBtnFg); + mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='black' stroke-width='1.75'%3E%3Cpath d='M3 19a2 2 0 0 1-1-2V2a2 2 0 0 1 1-1h13a2 2 0 0 1 2 1'/%3E%3Crect x='6' y='5' width='16' height='18' rx='1.5' ry='1.5'/%3E%3C/svg%3E"); + mask-repeat: no-repeat; + margin: 0.475rem; + line-height: 0; +} + +.highlight .copy button:focus::after, +.highlight .copy button:active::after { + display: inline-block; + content: ""; + mask: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='22' height='22' viewBox='0 0 24 24' stroke-width='1.25' stroke='black' fill='none' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath stroke='none' d='M0 0h24v24H0z' fill='none'%3E%3C/path%3E%3Cpath d='M5 12l5 5l10 -10'%3E%3C/path%3E%3C/svg%3E") no-repeat 50% 50%; + mask-size: cover; + margin: 0.2375rem; +} + +.highlight .copy button:hover, +.highlight .copy button:focus:focus-visible { + opacity: 1; +} + +.highlight .copy button:hover div, +.highlight .copy button:focus:focus-visible div { + opacity: var(--ec-frm-inlBtnBgHoverOrFocusOpa); +} + +.highlight .copy button:active { + opacity: 1; +} + +.highlight .copy button:active div { + opacity: var(--ec-frm-inlBtnBgActOpa); +} + +@media (hover: hover) { + .highlight .copy button { + opacity: 0; + width: 2rem; + height: 2rem; + } + + .highlight:hover .copy button:not(:hover) { + opacity: 0.75; + } +} diff --git a/layouts/_default/_markup/render-codeblock.html b/layouts/_default/_markup/render-codeblock.html index 72eda41..96e2b57 100644 --- a/layouts/_default/_markup/render-codeblock.html +++ b/layouts/_default/_markup/render-codeblock.html @@ -3,7 +3,9 @@ In production, scripts/highlight.mjs post-processes all codeblocks with Shiki, so do not transform the codeblock here. */ -}} - 
{{ .Inner }}
+
+ 
{{ .Inner }}
+
{{- else }} {{- /* In development, fall back on the faster default highlighting built in to Doks.