Support transient dependencies when bundling (whitelisting) node modules

lib/packExternalModules.js

@@ -6,21 +6,32 @@ const path = require('path');
 const childProcess = require('child_process');
 const fse = require('fs-extra');
 const npm = require('npm-programmatic');
+const isBuiltinModule = require('is-builtin-module');
 
-function getProdModules(externalModules, packagePath) {
-  const packageJson = require(path.join(process.cwd(), packagePath));
+function getProdModules(externalModules, packagePath, dependencyGraph) {
+  const packageJsonPath = path.join(process.cwd(), packagePath);
+  const packageJson = require(packageJsonPath);
   const prodModules = [];
 
   // only process the module stated in dependencies section
   if (!packageJson.dependencies) {
     return [];
   }
 
+  // Get versions of all transient modules
   _.forEach(externalModules, module => {
-    const moduleVersion = packageJson.dependencies[module];
+    let moduleVersion = packageJson.dependencies[module.external];
 
     if (moduleVersion) {
-      prodModules.push(`${module}@${moduleVersion}`);
+      prodModules.push(`${module.external}@${moduleVersion}`);
+    } else if (!_.has(packageJson, `devDependencies.${module.external}`)) {
+      // Add transient dependencies if they appear not in the service's dev dependencies
+      const originInfo = _.get(dependencyGraph, `dependencies.${module.origin}`, {});
+      moduleVersion = _.get(originInfo, `dependencies.${module.external}.version`);
+      if (!moduleVersion) {
+        this.serverless.cli.log(`WARNING: Could not determine version of module ${module.external}`);
+      }
+      prodModules.push(moduleVersion ? `${module.external}@${moduleVersion}` : module.external);
     }
   });
 
@@ -41,19 +52,32 @@ function getExternalModuleName(module) {
 }
 
 function isExternalModule(module) {
-  return _.startsWith(module.identifier(), 'external ');
+  return _.startsWith(module.identifier(), 'external ') && !isBuiltinModule(getExternalModuleName(module));
 }
 
-function getExternalModules(stats) {
+/**
+ * Find the original module that required the transient dependency. Returns
+ * undefined if the module is a first level dependency.
+ * @param {Object} issuer - Module issuer
+ */
+function findExternalOrigin(issuer) {
+  if (!_.isNil(issuer) && _.startsWith(issuer.rawRequest, './')) {
+    return findExternalOrigin(issuer.issuer);
+  }
+  return issuer;
+}
 
+function getExternalModules(stats) {
   const externals = new Set();
 
   _.forEach(stats.compilation.chunks, chunk => {
     // Explore each module within the chunk (built inputs):
     _.forEach(chunk.modules, module => {
-      // Explore each source file path that was included into the module:
       if (isExternalModule(module)) {
-        externals.add(getExternalModuleName(module));
+        externals.add({
+          origin: _.get(findExternalOrigin(module.issuer), 'rawRequest'),
+          external: getExternalModuleName(module)
+        });
       }
     });
   });
@@ -87,11 +111,28 @@ module.exports = {
     }
 
     const packagePath = includes.packagePath || './package.json';
+    const packageJsonPath = path.join(process.cwd(), packagePath);
+
+    this.options.verbose && this.serverless.cli.log(`Fetch dependency graph from ${packageJsonPath}`);
+    // Get first level dependency graph
+    const command = 'npm ls -prod -json -depth=1';  // Only prod dependencies
+    let dependencyGraph = {};
+    try {
+      const depJson = childProcess.execSync(command, {
+        cwd: path.dirname(packageJsonPath),
+        maxBuffer: this.serverless.service.custom.packExternalModulesMaxBuffer || 200 * 1024,
+        encoding: 'utf8'
+      });
+      dependencyGraph = JSON.parse(depJson);
+    } catch (e) {
+      // We rethrow here. It's not recoverable.
+      throw e;
+    }
 
     // (1) Generate dependency composition
     const compositeModules = _.uniq(_.flatMap(stats.stats, compileStats => {
-      const externalModules = getExternalModules(compileStats);
-      return getProdModules(externalModules, packagePath);
+      const externalModules = getExternalModules.call(this, compileStats);
+      return getProdModules.call(this, externalModules, packagePath, dependencyGraph);
     }));
 
     // (1.a) Install all needed modules
@@ -122,7 +163,7 @@ module.exports = {
       const modulePackage = {
         dependencies: {}
       };
-      const prodModules = getProdModules(getExternalModules(compileStats), packagePath);
+      const prodModules = getProdModules.call(this, getExternalModules.call(this, compileStats), packagePath, dependencyGraph);
       _.forEach(prodModules, prodModule => {
         const splitModule = _.split(prodModule, '@');
         // If we have a scoped module we have to re-add the @

package.json

@@ -31,6 +31,7 @@
     "bluebird": "^3.4.0",
     "fs-extra": "^0.26.7",
     "glob": "^7.1.2",
+    "is-builtin-module": "^1.0.0",
     "lodash": "^4.17.4",
     "npm-programmatic": "^0.0.7",
     "semver": "^5.4.1",

tests/mocks/child_process.mock.js

@@ -7,6 +7,7 @@
 module.exports.create = sandbox => {
   const childProcessMock = {
     exec: sandbox.stub().yields(),
+    execSync: sandbox.stub().returns('{}')
   };
 
   return childProcessMock;

tests/packExternalModules.test.js

@@ -143,6 +143,7 @@ describe('packExternalModules', () => {
       npmMock.install.returns(BbPromise.resolve());
       fsExtraMock.copy.yields();
       childProcessMock.exec.yields();
+      childProcessMock.execSync.returns('{}');
       return expect(module.packExternalModules(stats)).to.be.fulfilled
       .then(() => BbPromise.all([
         // npm install should have been called with all externals from the package mock
@@ -154,6 +155,7 @@ describe('packExternalModules', () => {
         ],
         {
           cwd: 'outputPath/dependencies',
+          maxBuffer: 204800,
           save: true
         }),
         // The module package JSON and the composite one should have been stored