From edc3d10417ccae03d936ac076f56890b5f761899 Mon Sep 17 00:00:00 2001 From: Mohamed Elkholy Date: Mon, 8 Mar 2021 20:46:28 +0200 Subject: [PATCH] feat(plugin): add event `ext.configure` that allows tasks to change processing options for different extensions --- src/__tests__/__snapshots__/postbuild.js.snap | 24 ++++---- src/__tests__/postbuild.js | 22 +++++--- src/files/generic.ts | 4 +- src/filesystem.ts | 18 +++--- src/options.ts | 55 +++++++++---------- src/postbuild.ts | 36 ++++++------ src/tasks.ts | 33 ++++++----- src/tasks/purgecss/index.ts | 10 +++- test/__fixtures__/options.js | 24 +++++--- 9 files changed, 124 insertions(+), 102 deletions(-) diff --git a/src/__tests__/__snapshots__/postbuild.js.snap b/src/__tests__/__snapshots__/postbuild.js.snap index e885b95..1a5f82a 100644 --- a/src/__tests__/__snapshots__/postbuild.js.snap +++ b/src/__tests__/__snapshots__/postbuild.js.snap @@ -33,12 +33,12 @@ exports[`init options schema should invalidate empty options 1`] = `false`; exports[`init options schema should invalidate empty options 2`] = ` Array [ "\\"ignore[0]\\" is not allowed to be empty", + "\\"processing.concurrency\\" must be greater than or equal to 1", + "\\"processing.strategy\\" must be one of [sequential, parallel]", + "\\"processing.strategy\\" is not allowed to be empty", "\\"extensions.txt.concurrency\\" must be greater than or equal to 1", - "\\"extensions.txt.strategy\\" must be one of [steps, parallel]", + "\\"extensions.txt.strategy\\" must be one of [sequential, parallel]", "\\"extensions.txt.strategy\\" is not allowed to be empty", - "\\"defaultStrategy\\" must be one of [steps, parallel]", - "\\"defaultStrategy\\" is not allowed to be empty", - "\\"defaultConcurrency\\" must be greater than or equal to 1", ] `; @@ -49,10 +49,10 @@ Array [ "\\"events.on.load\\" is not allowed", "\\"events.html.content\\" is not allowed", "\\"events.txt.parse\\" is not allowed", + "\\"processing.concurrency\\" must be greater than or equal to 1", + "\\"processing.strategy\\" must be one of [sequential, parallel]", "\\"extensions.txt.concurrency\\" must be greater than or equal to 1", - "\\"extensions.txt.strategy\\" must be one of [steps, parallel]", - "\\"defaultStrategy\\" must be one of [steps, parallel]", - "\\"defaultConcurrency\\" must be greater than or equal to 1", + "\\"extensions.txt.strategy\\" must be one of [sequential, parallel]", ] `; @@ -76,13 +76,13 @@ Array [ "\\"events.html.parse\\" must be of type function", "\\"events.foo\\" must be of type object", "\\"events.txt.content\\" must be of type function", + "\\"processing.concurrency\\" must be a number", + "\\"processing.strategy\\" must be one of [sequential, parallel]", + "\\"processing.strategy\\" must be a string", "\\"extensions.foo\\" must be of type object", "\\"extensions.txt.concurrency\\" must be a number", - "\\"extensions.txt.strategy\\" must be one of [steps, parallel]", + "\\"extensions.txt.strategy\\" must be one of [sequential, parallel]", "\\"extensions.txt.strategy\\" must be a string", - "\\"defaultStrategy\\" must be one of [steps, parallel]", - "\\"defaultStrategy\\" must be a string", - "\\"defaultConcurrency\\" must be a number", ] `; @@ -197,7 +197,7 @@ Array [ ] `; -exports[`run correctly runs steps strat 1`] = ` +exports[`run correctly runs sequential strat 1`] = ` Array [ Array [ "Loaded 1/4 Processed 0/4 Wrote 0/4", diff --git a/src/__tests__/postbuild.js b/src/__tests__/postbuild.js index d169e44..d2ba4a6 100644 --- a/src/__tests__/postbuild.js +++ b/src/__tests__/postbuild.js @@ -44,7 +44,7 @@ describe('bootstrap', () => { const tasks = { register: jest.fn(), setOptions: jest.fn(), - run: jest.fn() + run: jest.fn().mockImplementation(() => Promise.resolve()) } const filesystem = { setRoot: jest.fn() @@ -105,11 +105,11 @@ describe('run', () => { foo: ['file1.foo', 'file2.foo'], bar: ['file1.bar', 'file2/file2.bar'] }), - run: jest.fn() + run: jest.fn().mockImplementation(() => Promise.resolve()) } const setStatus = jest.fn() const filesystem = { - create: jest.fn(), + create: jest.fn().mockImplementation(() => Promise.resolve()), reporter: { getReports: jest.fn().mockImplementation(() => ['report1', 'report2']), getTotalSaved: jest.fn().mockImplementation(() => [5, 5]) @@ -159,22 +159,26 @@ describe('run', () => { { title: 'correctly runs parallel strat', options: { - defaultStrategy: 'parallel', - defaultConcurrency: 1 + processing: { + strategy: 'parallel', + concurrency: 1 + } } }, { - title: 'correctly runs steps strat', + title: 'correctly runs sequential strat', options: { - defaultStrategy: 'steps', - defaultConcurrency: 1 + processing: { + strategy: 'sequential', + concurrency: 1 + } } }, { title: 'correctly runs mixed strats', options: { extensions: { - foo: { strategy: 'steps', concurrency: 1 }, + foo: { strategy: 'sequential', concurrency: 1 }, bar: { strategy: 'parallel', concurrency: 1 } } } diff --git a/src/files/generic.ts b/src/files/generic.ts index 841c056..58080cf 100644 --- a/src/files/generic.ts +++ b/src/files/generic.ts @@ -2,7 +2,7 @@ import { Promise } from 'bluebird' import { File } from './base' /** - * Handles files not known to the plugin + * Handles files with unknown extensions */ export class FileGeneric extends File { /** @@ -11,7 +11,7 @@ export class FileGeneric extends File { */ read (): Promise { return this.file.read() - .then(raw => this.emit('glob', 'content', { + .then(raw => this.emit('unknown', 'content', { ...this.emitPayload(), raw }, 'raw')) diff --git a/src/filesystem.ts b/src/filesystem.ts index f88dbc8..cca2503 100644 --- a/src/filesystem.ts +++ b/src/filesystem.ts @@ -3,8 +3,9 @@ import { promises as fs } from 'fs' import path from 'path' import glob from 'glob' import filesize from 'filesize' -import { IOptions } from './options' +import { toInteger } from 'lodash' import { PostbuildError } from '~/common' +import type { IOptions } from './options' const globAsync = Promise.promisify(glob) as typeof glob.__promisify__ /** @@ -47,14 +48,12 @@ export class FilesystemReport { getConsoleOutput (): string { const saved = this.size[1] !== undefined - ? (((this.size[0] - this.size[1]) / this.size[1]) * 100) - .toFixed() - .replace('-0', '0') - : '0' + ? toInteger(((this.size[0] - this.size[1]) / this.size[1]) * 100) + : 0 return [ colorize.tag(this.tag), colorize.file(this.file), - colorize.size(formatSize(this.size[0]) + (saved !== '0' ? ` ${(saved)}%` : '')), + colorize.size(formatSize(this.size[0]) + (saved !== 0 ? ` ${(saved)}%` : '')), Object.keys(this.meta).map(field => colorize.meta(field, this.meta[field])) ].flat().join(' ') } @@ -116,7 +115,8 @@ export class FilesystemReporter { */ export class Filesystem { /** - * Absolute ath to `/public` directory + * Absolute path to be used for resolving relative paths passed to class methods + * This should point to `/public` directory */ root: string = '' @@ -128,9 +128,7 @@ export class Filesystem { } /** - * Sets root path for `/public` - * - * @param root - Absolute path to `/public` + * Sets root path */ setRoot (root: string): void { this.root = root diff --git a/src/options.ts b/src/options.ts index 1243630..870ee08 100644 --- a/src/options.ts +++ b/src/options.ts @@ -1,6 +1,14 @@ import { GatsbyJoi } from './gatsby' import { ITaskApiEvents, ITaskOptions } from './tasks' +// Available extension processing strategies +export type IOptionProcessingStrategy = 'sequential' | 'parallel' +// Processing options interface +export interface IOptionProcessing { + concurrency: number + strategy: IOptionProcessingStrategy +} + /** * Plugin options interface */ @@ -10,21 +18,14 @@ export type IOptions = { consoleReport: boolean ignore: string[] events: ITaskApiEvents - defaultConcurrency: number - defaultStrategy: IOptionsExtStrategy + processing: IOptionProcessing extensions: { - [ext: string]: { - concurrency?: number - strategy?: IOptionsExtStrategy - } | undefined + [ext: string]: Partial | undefined } } & { [task: string]: ITaskOptions } -// Available extension processing strategies -export type IOptionsExtStrategy = 'steps' | 'parallel' - /** * Default values for plugin options */ @@ -35,19 +36,24 @@ export const DEFAULTS: IOptions = { consoleReport: true, ignore: [], events: {}, - defaultStrategy: 'parallel', - defaultConcurrency: 10, - extensions: { - html: { - strategy: 'steps' - } - } + processing: { + strategy: 'parallel', + concurrency: 10 + }, + extensions: {} } /** * Plugin options schema */ export function schema (joi: GatsbyJoi): GatsbyJoi { + const processingSchema = joi.object({ + concurrency: joi.number().min(1) + .description('How many files to process at once.'), + strategy: joi.string() + .valid('sequential', 'parallel') + .description('Determines how the files are processed.') + }) return joi.object({ enabled: joi.boolean() .description('Whether to run the postbuild or not.'), @@ -70,18 +76,9 @@ export function schema (joi: GatsbyJoi): GatsbyJoi { content: joi.function() })) .description('Set of events to added as a custom postbuild task.'), - extensions: joi.object().pattern(joi.string(), joi.object({ - concurrency: joi.number().min(1) - .description('How many files to process at once.'), - strategy: joi.string() - .valid('steps', 'parallel') - .description('Determines how the files are processed.') - })) - .description('Changes how files of a specific extension are processed.'), - defaultStrategy: joi.string() - .valid('steps', 'parallel') - .description('Determines how the files are processed.'), - defaultConcurrency: joi.number().min(1) - .description('How many files to process at once.') + processing: processingSchema + .description('Default file processing options for all extensions.'), + extensions: joi.object().pattern(joi.string(), processingSchema) + .description('Changes how files of a specific extension are processed.') }) } diff --git a/src/postbuild.ts b/src/postbuild.ts index a0e5cb4..724aa3f 100644 --- a/src/postbuild.ts +++ b/src/postbuild.ts @@ -4,7 +4,7 @@ import _ from 'lodash' import { Filesystem } from './filesystem' import { Tasks, ITask, ITaskOptions } from './tasks' import { File } from './files' -import { DEFAULTS, schema, IOptions } from './options' +import { DEFAULTS, schema, IOptions, IOptionProcessing } from './options' import { ERROR_MAP, debug } from './common' import type { GatsbyJoi, GatsbyNodeArgs, GatsbyPluginOptions } from './gatsby' @@ -12,7 +12,7 @@ import type { GatsbyJoi, GatsbyNodeArgs, GatsbyPluginOptions } from './gatsby' * Interface for the main postbuild object thats is passed * to all event callbacks */ -export type IPostbuildArgs = { +export type IPostbuildArgs = { /** * Current active task */ @@ -157,21 +157,13 @@ export default class Postbuild { } }) - // File processing options - const defaultConcLimit = this.options.defaultConcurrency - const defaultStrategy = this.options.defaultStrategy - const extConfig = this.options.extensions - /** - * Runs file events for a specific extension - * parallel can be set to true to process files in parallel + * Processes files of a given extension using the given processing options */ - async function processFiles (ext: string): Promise { - const strat = extConfig[ext]?.strategy ?? defaultStrategy - const conc = { - concurrency: extConfig[ext]?.concurrency ?? defaultConcLimit - } - debug(`Processing ${files[ext].length} "${ext}" files with`, { strat, conc }) + async function processFiles (ext: string, options: IOptionProcessing): Promise { + debug(`Processing ${files[ext].length} files with extension "${ext}" using`, options) + const strat = options.strategy + const conc = { concurrency: options.concurrency } // Process all files at one step all at the same time if (strat === 'parallel') { await Promise.map(files[ext], (file, i) => { @@ -193,7 +185,19 @@ export default class Postbuild { } // Run one extension at a time - await Promise.each(Object.keys(files), ext => processFiles(ext)) + await Promise.each(Object.keys(files), ext => { + // Extension processing options + const config = { ...this.options.processing } + return this.tasks.run(ext as 'unknown', 'configure', { + file: undefined, + filesystem: this.fs, + gatsby, + config + }).then(() => processFiles(ext, { + ...config, + ...this.options.extensions[ext] + })) + }) // Write the full postbuild report const reports = this.fs.reporter.getReports() diff --git a/src/tasks.ts b/src/tasks.ts index e3fd98b..16b03f9 100644 --- a/src/tasks.ts +++ b/src/tasks.ts @@ -4,7 +4,7 @@ import { Filesystem } from './filesystem' import { createDebug, PostbuildError } from './common' import type { File, FileGeneric, FileHtml } from './files' import type { Node as parse5Node } from 'parse5' -import type { IOptions } from './options' +import type { IOptions, IOptionProcessing } from './options' import type { GatsbyJoi } from './gatsby' import type { IPostbuildArgs } from './postbuild' const debug = createDebug('tasks') @@ -28,30 +28,32 @@ type Keys = { [K in keyof O]: O[K] extends C ? K : never }[keyof O] * Interface for an event callback */ type IEvent< - F extends File | undefined, O extends ITaskOptions, P extends Object = {}, + F extends File | undefined = undefined, R = void -> = Fn<[IPostbuildArgs], R> +> = Fn<[IPostbuildArgs], R> /** * Defines every event api within the plugin */ export interface IEvents { on: { - bootstrap: IEvent - postbuild: IEvent - shutdown: IEvent + bootstrap: IEvent + postbuild: IEvent + shutdown: IEvent } html: { - parse: IEvent - tree: IEvent - node: IEvent - serialize: IEvent - write: IEvent + configure: IEvent + parse: IEvent + tree: IEvent + node: IEvent + serialize: IEvent + write: IEvent } - glob: { - content: IEvent + unknown: { + configure: IEvent + content: IEvent } } @@ -79,9 +81,9 @@ export interface ITaskOptions { * @see https://github.com/microsoft/TypeScript/pull/41524 */ export type ITaskApiEvents = { - [K in Exclude]?: Partial[K]> + [K in Exclude]?: Partial[K]> } & { - [glob: string]: Partial['glob']> + [glob: string]: Partial['unknown']> } /** @@ -293,6 +295,7 @@ export class Tasks { if (file !== undefined) { const fileEvents = this.fileEvents[file.relative] ?? [] for (const fe of fileEvents) { + // no need to check for event type here if (event in fe[0].api.events[fe[1]]) events.push(fe) } } else { diff --git a/src/tasks/purgecss/index.ts b/src/tasks/purgecss/index.ts index 6e36093..43c37f8 100644 --- a/src/tasks/purgecss/index.ts +++ b/src/tasks/purgecss/index.ts @@ -38,10 +38,14 @@ class DIContainer { getFile (file: FileHtml): HtmlTransformer { return this.files[file.relative] } + + deleteFile (file: FileHtml): void { + delete this.files[file.relative] + } } let di: DIContainer -// @ts-expect-error + export const events: ITaskApiEvents = { on: { postbuild: ({ filesystem, options }) => { @@ -49,6 +53,9 @@ export const events: ITaskApiEvents = { } }, html: { + configure: ({ config }) => { + config.strategy = 'sequential' + }, tree: ({ file }) => { di.createFile(file) return di.getFile(file).load() @@ -58,6 +65,7 @@ export const events: ITaskApiEvents = { }, serialize: ({ file }) => { return di.getFile(file).purgeStyles() + .then(() => di.deleteFile(file)) } } } diff --git a/test/__fixtures__/options.js b/test/__fixtures__/options.js index 1328114..b00bb81 100644 --- a/test/__fixtures__/options.js +++ b/test/__fixtures__/options.js @@ -13,8 +13,10 @@ export default [ options: { enabled: 5, report: '', - defaultStrategy: 5, - defaultConcurrency: false, + processing: { + strategy: 5, + concurrency: false + }, ignore: {}, events: { foo: 'invalid', @@ -34,8 +36,10 @@ export default [ { title: 'should invalidate empty options', options: { - defaultStrategy: '', - defaultConcurrency: 0, + processing: { + strategy: '', + concurrency: 0 + }, ignore: [''], extensions: { txt: { @@ -48,8 +52,10 @@ export default [ { title: 'should invalidate invalid options', options: { - defaultConcurrency: -1, - defaultStrategy: 'foo', + processing: { + strategy: 'foo', + concurrency: -1 + }, extensions: { txt: { concurrency: -1, @@ -70,8 +76,10 @@ export default [ report: false, consoleReport: false, ignore: ['a'], - defaultConcurrency: 25, - defaultStrategy: 'steps', + processing: { + strategy: 'sequential', + concurrency: 25 + }, extensions: { txt: { concurrency: 11,