Merge pull request #829 from timocov/fix825

Added cache for some FS operations while compile

Author: johnnyreilly

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 5.1.0
+
+* [feat: Added cache for some FS operations while compiling - `experimentalFileCaching`](https://github.com/TypeStrong/ts-loader/pull/829) - thanks @timocov!
+
 ## 5.0.0
 
 * [feat: Fixed issue with incorrect output path for declaration files](https://github.com/TypeStrong/ts-loader/pull/822) - thanks @JonWallsten! **BREAKING CHANGE**

README.md

@@ -543,20 +543,14 @@ Extending `tsconfig.json`:
 
 Note that changes in the extending file while not be respected by `ts-loader`. Its purpose is to satisfy the code editor.
 
-### `LoaderOptionsPlugin`
+### experimentalFileCaching _(boolean) (default=false)_
 
-[There's a known "gotcha"](https://github.com/TypeStrong/ts-loader/issues/283) if you are using webpack 2 with the `LoaderOptionsPlugin`. If you are faced with the `Cannot read property 'unsafeCache' of undefined` error then you probably need to supply a `resolve` object as below: (Thanks @jeffijoe!)
+By default whenever the TypeScript compiler needs to check that a file/directory exists or resolve symlinks it makes syscalls.
+It does not cache the result of these operations and this may result in many syscalls with the same arguments ([see comment](https://github.com/TypeStrong/ts-loader/issues/825#issue-354725524) with example).
+In some cases it may produce performance degradation.
 
-```js
-new LoaderOptionsPlugin({
-  debug: false,
-  options: {
-    resolve: {
-      extensions: [".ts", ".tsx", ".js"]
-    }
-  }
-});
-```
+This flag enables caching for some FS-functions like `fileExists`, `realpath` and `directoryExists` for TypeScript compiler.
+Note that caches are cleared between compilations.
 
 ### Usage with Webpack watch
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ts-loader",
-  "version": "5.0.0",
+  "version": "5.1.0",
   "description": "TypeScript loader for webpack",
   "main": "index.js",
   "types": "dist/types/index.d.ts",

src/index.ts

@@ -157,7 +157,8 @@ const validLoaderOptions: ValidLoaderOptions[] = [
   'getCustomTransformers',
   'reportFiles',
   'experimentalWatchApi',
-  'allowTsInNodeModules'
+  'allowTsInNodeModules',
+  'experimentalFileCaching'
 ];
 
 /**
@@ -210,7 +211,8 @@ function makeLoaderOptions(instanceName: string, loaderOptions: LoaderOptions) {
       reportFiles: [],
       // When the watch API usage stabilises look to remove this option and make watch usage the default behaviour when available
       experimentalWatchApi: false,
-      allowTsInNodeModules: false
+      allowTsInNodeModules: false,
+      experimentalFileCaching: false
     } as Partial<LoaderOptions>,
     loaderOptions
   );

src/instances.ts

@@ -247,6 +247,12 @@ function successfulTypeScriptInstance(
     colors
   });
 
+  if (!loader._compiler.hooks) {
+    throw new Error(
+      "You may be using an old version of webpack; please check you're using at least version 4"
+    );
+  }
+
   if (loaderOptions.experimentalWatchApi && compiler.createWatchProgram) {
     log.logInfo('Using watch api');
 
@@ -266,17 +272,22 @@ function successfulTypeScriptInstance(
       .getProgram()
       .getProgram();
   } else {
-    const servicesHost = makeServicesHost(scriptRegex, log, loader, instance);
+    const servicesHost = makeServicesHost(
+      scriptRegex,
+      log,
+      loader,
+      instance,
+      loaderOptions.experimentalFileCaching
+    );
+
     instance.languageService = compiler.createLanguageService(
-      servicesHost,
+      servicesHost.servicesHost,
       compiler.createDocumentRegistry()
     );
-  }
 
-  if (!loader._compiler.hooks) {
-    throw new Error(
-      "You may be using an old version of webpack; please check you're using at least version 4"
-    );
+    if (servicesHost.clearCache !== null) {
+      loader._compiler.hooks.watchRun.tap('ts-loader', servicesHost.clearCache);
+    }
   }
 
   loader._compiler.hooks.afterCompile.tapAsync(

src/interfaces.ts

@@ -303,6 +303,7 @@ export interface LoaderOptions {
     | (() => typescript.CustomTransformers | undefined);
   experimentalWatchApi: boolean;
   allowTsInNodeModules: boolean;
+  experimentalFileCaching: boolean;
 }
 
 export interface TSFile {

src/servicesHost.ts

@@ -14,15 +14,23 @@ import {
   Webpack
 } from './interfaces';
 
+export type Action = () => void;
+
+export interface ServiceHostWhichMayBeCacheable {
+  servicesHost: typescript.LanguageServiceHost;
+  clearCache: Action | null;
+}
+
 /**
  * Create the TypeScript language service
  */
 export function makeServicesHost(
   scriptRegex: RegExp,
   log: logger.Logger,
   loader: Webpack,
-  instance: TSInstance
-) {
+  instance: TSInstance,
+  enableFileCaching: boolean
+): ServiceHostWhichMayBeCacheable {
   const {
     compiler,
     compilerOptions,
@@ -52,9 +60,12 @@ export function makeServicesHost(
   const moduleResolutionHost: ModuleResolutionHost = {
     fileExists,
     readFile: readFileWithFallback,
-    realpath: compiler.sys.realpath
+    realpath: compiler.sys.realpath,
+    directoryExists: compiler.sys.directoryExists
   };
 
+  const clearCache = enableFileCaching ? addCache(moduleResolutionHost) : null;
+
   // loader.context seems to work fine on Linux / Mac regardless causes problems for @types resolution on Windows for TypeScript < 2.3
   const getCurrentDirectory = () => loader.context;
 
@@ -97,13 +108,15 @@ export function makeServicesHost(
     /**
      * For @types expansion, these two functions are needed.
      */
-    directoryExists: compiler.sys.directoryExists,
+    directoryExists: moduleResolutionHost.directoryExists,
 
     useCaseSensitiveFileNames: () => compiler.sys.useCaseSensitiveFileNames,
 
+    realpath: moduleResolutionHost.realpath,
+
     // The following three methods are necessary for @types resolution from TS 2.4.1 onwards see: https://github.com/Microsoft/TypeScript/issues/16772
-    fileExists: compiler.sys.fileExists,
-    readFile: compiler.sys.readFile,
+    fileExists: moduleResolutionHost.fileExists,
+    readFile: moduleResolutionHost.readFile,
     readDirectory: compiler.sys.readDirectory,
 
     getCurrentDirectory,
@@ -137,7 +150,7 @@ export function makeServicesHost(
     getCustomTransformers: () => instance.transformers
   };
 
-  return servicesHost;
+  return { servicesHost, clearCache };
 }
 
 /**
@@ -509,3 +522,49 @@ function populateDependencyGraphs(
     ] = true;
   });
 }
+
+type CacheableFunction = Extract<
+  keyof typescript.ModuleResolutionHost,
+  'fileExists' | 'directoryExists' | 'realpath'
+>;
+const cacheableFunctions: CacheableFunction[] = [
+  'fileExists',
+  'directoryExists',
+  'realpath'
+];
+
+function addCache(servicesHost: typescript.ModuleResolutionHost) {
+  const clearCacheFunctions: Action[] = [];
+
+  cacheableFunctions.forEach((functionToCache: CacheableFunction) => {
+    const originalFunction = servicesHost[functionToCache];
+    if (originalFunction !== undefined) {
+      const cache = createCache<ReturnType<typeof originalFunction>>(originalFunction);
+      servicesHost[
+        functionToCache
+      ] = cache.getCached as typescript.ModuleResolutionHost[CacheableFunction];
+      clearCacheFunctions.push(cache.clear);
+    }
+  });
+
+  return () => clearCacheFunctions.forEach(clear => clear());
+}
+
+function createCache<TOut>(originalFunction: (arg: string) => TOut) {
+  const cache = new Map<string, TOut>();
+  return {
+    clear: () => {
+      cache.clear();
+    },
+    getCached: (arg: string) => {
+      let res = cache.get(arg);
+      if (res !== undefined) {
+        return res;
+      }
+
+      res = originalFunction(arg);
+      cache.set(arg, res);
+      return res;
+    }
+  };
+}

test/.prettierrc

@@ -0,0 +1,10 @@
+{
+  "overrides": [
+    {
+      "files": "*.js",
+      "options": {
+        "singleQuote": true
+      }
+    }
+  ]
+}