refactor: restructure dependency traversal files (#708)

* refactor: remove getSrcFilesAndExternalModules

* refactor: move getNewCache to util file

* refactor: change signature of getDependencyNamesAndPathsForDependencies

* refactor: remove intermediate exports

* refactor: move getDependencyNamesAndPathsForDependencies

* refactor: rename functions

* refactor: remove unnecessary export

* refactor: move getPluginsModulesPath to util file

* refactor: rename function

src/main.js

@@ -3,7 +3,7 @@ const { extname } = require('path')
 require('./utils/polyfills')
 const { getFlags } = require('./feature_flags')
 const { getFunctionsFromPaths } = require('./runtimes')
-const { getPluginsModulesPath } = require('./runtimes/node/bundlers/zisi')
+const { getPluginsModulesPath } = require('./runtimes/node/utils/plugin_modules_path')
 const { listFunctionsDirectories, resolveFunctionsDirectories } = require('./utils/fs')
 const { zipFunction, zipFunctions } = require('./zip')
 

src/runtimes/node/bundlers/esbuild/additional_files.js

@@ -0,0 +1,52 @@
+const { getPackageJson } = require('../../utils/package_json')
+const { getNewCache } = require('../../utils/traversal_cache')
+const { getDependencyPathsForDependency } = require('../zisi/traverse')
+
+const getSrcFilesForDependencies = async function ({
+  dependencies: dependencyNames,
+  basedir,
+  state = getNewCache(),
+  pluginsModulesPath,
+}) {
+  if (dependencyNames.length === 0) {
+    return []
+  }
+
+  const packageJson = await getPackageJson(basedir)
+  const dependencies = await Promise.all(
+    dependencyNames.map((dependencyName) =>
+      getSrcFilesForDependency({
+        dependency: dependencyName,
+        basedir,
+        state,
+        packageJson,
+        pluginsModulesPath,
+      }),
+    ),
+  )
+  const paths = new Set(dependencies.flat())
+
+  return [...paths]
+}
+
+const getSrcFilesForDependency = async function ({
+  dependency,
+  basedir,
+  state = getNewCache(),
+  packageJson,
+  pluginsModulesPath,
+}) {
+  try {
+    const paths = await getDependencyPathsForDependency({ dependency, basedir, state, packageJson, pluginsModulesPath })
+
+    return paths
+  } catch (error) {
+    if (error.code === 'MODULE_NOT_FOUND') {
+      return []
+    }
+
+    throw error
+  }
+}
+
+module.exports = { getSrcFilesForDependencies }

src/runtimes/node/bundlers/zisi/index.js

@@ -1,38 +1,21 @@
 const { dirname, basename, normalize } = require('path')
 
-const findUp = require('find-up')
 const { not: notJunk } = require('junk')
 const precinct = require('precinct')
 
 const { getPackageJson } = require('../../utils/package_json')
+const { getNewCache } = require('../../utils/traversal_cache')
 
 const { listImports } = require('./list_imports')
 const { resolvePathPreserveSymlinks } = require('./resolve')
-const {
-  getDependencyPathsForDependency,
-  getDependencyNamesAndPathsForDependencies,
-  getDependencyNamesAndPathsForDependency,
-  getNewCache,
-} = require('./traverse')
+const { getDependencyPathsForDependency } = require('./traverse')
 const { getTreeFiles } = require('./tree_files')
 const { shouldTreeShake } = require('./tree_shake')
 
-const AUTO_PLUGINS_DIR = '.netlify/plugins/'
-
-const getPluginsModulesPath = (srcDir) => findUp(`${AUTO_PLUGINS_DIR}node_modules`, { cwd: srcDir, type: 'directory' })
-
 // Retrieve the paths to the Node.js files to zip.
 // We only include the files actually needed by the function because AWS Lambda
 // has a size limit for the zipped file. It also makes cold starts faster.
-const listFilesUsingLegacyBundler = async function ({
-  featureFlags,
-  srcPath,
-  mainFile,
-  name,
-  srcDir,
-  stat,
-  pluginsModulesPath,
-}) {
+const listFiles = async function ({ featureFlags, srcPath, mainFile, name, srcDir, stat, pluginsModulesPath }) {
   const [treeFiles, depFiles] = await Promise.all([
     getTreeFiles(srcPath, stat),
     getDependencies({ featureFlags, functionName: name, mainFile, pluginsModulesPath, srcDir }),
@@ -165,9 +148,5 @@ const getTreeShakedDependencies = async function ({
 }
 
 module.exports = {
-  getDependencyPathsForDependency,
-  getDependencyNamesAndPathsForDependencies,
-  getDependencyNamesAndPathsForDependency,
-  getPluginsModulesPath,
-  listFilesUsingLegacyBundler,
+  listFiles,
 }

src/runtimes/node/bundlers/zisi/traverse.js

@@ -1,7 +1,6 @@
 const { dirname } = require('path')
 
 const { getModuleName } = require('../../utils/module')
-const { getPackageJson } = require('../../utils/package_json')
 
 const { getNestedDependencies, handleModuleNotFound } = require('./nested')
 const { getPublishedFiles } = require('./published')
@@ -10,9 +9,6 @@ const { getSideFiles } = require('./side_files')
 
 const EXCLUDED_MODULES = new Set(['aws-sdk'])
 
-// Local cache used for optimizing the traversal of module dependencies.
-const getNewCache = () => ({ localFiles: new Set(), moduleNames: new Set(), modulePaths: new Set() })
-
 // When a file requires a module, we find its path inside `node_modules` and
 // use all its published files. We also recurse on the module's dependencies.
 const getDependencyPathsForDependency = async function ({
@@ -37,66 +33,6 @@ const getDependencyPathsForDependency = async function ({
   }
 }
 
-const getDependencyNamesAndPathsForDependencies = async function ({
-  dependencies: dependencyNames,
-  basedir,
-  state = getNewCache(),
-  pluginsModulesPath,
-}) {
-  if (dependencyNames.length === 0) {
-    return {
-      moduleNames: [],
-      paths: [],
-    }
-  }
-
-  const packageJson = await getPackageJson(basedir)
-  const dependencies = await Promise.all(
-    dependencyNames.map((dependencyName) =>
-      getDependencyNamesAndPathsForDependency({
-        dependency: dependencyName,
-        basedir,
-        state,
-        packageJson,
-        pluginsModulesPath,
-      }),
-    ),
-  )
-  const moduleNames = new Set(dependencies.flatMap((dependency) => [...dependency.moduleNames]))
-  const paths = new Set(dependencies.flatMap((dependency) => [...dependency.paths]))
-
-  return {
-    moduleNames: [...moduleNames],
-    paths: [...paths],
-  }
-}
-
-const getDependencyNamesAndPathsForDependency = async function ({
-  dependency,
-  basedir,
-  state = getNewCache(),
-  packageJson,
-  pluginsModulesPath,
-}) {
-  try {
-    const paths = await getDependencyPathsForDependency({ dependency, basedir, state, packageJson, pluginsModulesPath })
-
-    return {
-      moduleNames: [...state.moduleNames],
-      paths,
-    }
-  } catch (error) {
-    if (error.code === 'MODULE_NOT_FOUND') {
-      return {
-        moduleNames: [],
-        paths: [],
-      }
-    }
-
-    throw error
-  }
-}
-
 const getDependenciesForModuleName = async function ({ moduleName, basedir, state, pluginsModulesPath }) {
   if (isExcludedModule(moduleName)) {
     return []
@@ -149,7 +85,4 @@ const getNestedModules = async function ({ modulePath, state, packageJson, plugi
 
 module.exports = {
   getDependencyPathsForDependency,
-  getDependencyNamesAndPathsForDependencies,
-  getDependencyNamesAndPathsForDependency,
-  getNewCache,
 }

src/runtimes/node/src_files.js

@@ -8,7 +8,8 @@ const pGlob = promisify(glob)
 
 const { JS_BUNDLER_ZISI } = require('../../utils/consts')
 
-const { getDependencyNamesAndPathsForDependencies, listFilesUsingLegacyBundler } = require('./bundlers/zisi')
+const { getSrcFilesForDependencies } = require('./bundlers/esbuild/additional_files')
+const zisiBundler = require('./bundlers/zisi')
 
 // Returns the subset of `paths` that don't match any of the glob expressions
 // from `exclude`.
@@ -57,35 +58,25 @@ const getPathsOfIncludedFiles = async (includedFiles, basePath) => {
   return { exclude, paths: [...new Set(normalizedPaths)] }
 }
 
-const getSrcFiles = async function ({ config, ...parameters }) {
-  const { paths } = await getSrcFilesAndExternalModules({
-    ...parameters,
-    externalNodeModules: config.externalNodeModules,
-  })
-
-  return paths
-}
-
-const getSrcFilesAndExternalModules = async function ({
+const getSrcFiles = async function ({
   bundler,
-  externalNodeModules = [],
+  config = {},
   featureFlags,
-  includedFiles = [],
-  includedFilesBasePath,
   mainFile,
   name,
   pluginsModulesPath,
   srcDir,
   srcPath,
   stat,
 }) {
+  const { externalNodeModules = [], includedFiles = [], includedFilesBasePath } = config
   const { exclude: excludedPaths, paths: includedFilePaths } = await getPathsOfIncludedFiles(
     includedFiles,
     includedFilesBasePath,
   )
 
   if (bundler === JS_BUNDLER_ZISI) {
-    const dependencyPaths = await listFilesUsingLegacyBundler({
+    const dependencyPaths = await zisiBundler.listFiles({
       featureFlags,
       srcPath,
       mainFile,
@@ -96,20 +87,17 @@ const getSrcFilesAndExternalModules = async function ({
     })
     const includedPaths = filterExcludedPaths([...dependencyPaths, ...includedFilePaths], excludedPaths)
 
-    return {
-      moduleNames: [],
-      paths: includedPaths,
-    }
+    return includedPaths
   }
 
-  const { moduleNames, paths: dependencyPaths } = await getDependencyNamesAndPathsForDependencies({
+  const dependencyPaths = await getSrcFilesForDependencies({
     dependencies: externalNodeModules,
     basedir: srcDir,
     pluginsModulesPath,
   })
   const includedPaths = filterExcludedPaths([...dependencyPaths, ...includedFilePaths], excludedPaths)
 
-  return { moduleNames, paths: [...includedPaths, mainFile] }
+  return [...includedPaths, mainFile]
 }
 
-module.exports = { getSrcFiles, getSrcFilesAndExternalModules }
+module.exports = { getSrcFiles }

src/runtimes/node/utils/plugin_modules_path.js

@@ -0,0 +1,7 @@
+const findUp = require('find-up')
+
+const AUTO_PLUGINS_DIR = '.netlify/plugins/'
+
+const getPluginsModulesPath = (srcDir) => findUp(`${AUTO_PLUGINS_DIR}node_modules`, { cwd: srcDir, type: 'directory' })
+
+module.exports = { getPluginsModulesPath }

src/runtimes/node/utils/traversal_cache.js

@@ -0,0 +1,4 @@
+// Local cache used for optimizing the traversal of module dependencies.
+const getNewCache = () => ({ localFiles: new Set(), moduleNames: new Set(), modulePaths: new Set() })
+
+module.exports = { getNewCache }

src/runtimes/node/zip_esbuild.js

@@ -6,7 +6,7 @@ const { zipNodeJs } = require('../../zip_node')
 
 const { bundleJsFile } = require('./bundlers/esbuild')
 const { getExternalAndIgnoredModulesFromSpecialCases } = require('./bundlers/esbuild/special_cases')
-const { getSrcFilesAndExternalModules } = require('./src_files')
+const { getSrcFiles } = require('./src_files')
 const { getBasePath } = require('./utils/base_path')
 
 const getFunctionBasePath = ({ basePathFromConfig, mainFile, supportingSrcFiles }) => {
@@ -70,11 +70,14 @@ const zipEsbuild = async ({
     srcFile: mainFile,
   })
   const bundlerWarnings = warnings.length === 0 ? undefined : warnings
-  const { paths: srcFiles } = await getSrcFilesAndExternalModules({
-    externalNodeModules: [...externalModules, ...Object.keys(nativeNodeModules)],
+  const srcFiles = await getSrcFiles({
     bundler: JS_BUNDLER_ESBUILD,
-    includedFiles: [...(config.includedFiles || []), ...additionalPaths],
-    includedFilesBasePath: config.includedFilesBasePath || basePath,
+    config: {
+      ...config,
+      externalNodeModules: [...externalModules, ...Object.keys(nativeNodeModules)],
+      includedFiles: [...(config.includedFiles || []), ...additionalPaths],
+      includedFilesBasePath: config.includedFilesBasePath || basePath,
+    },
     mainFile,
     name,
     srcPath,

src/runtimes/node/zip_zisi.js

@@ -3,7 +3,7 @@ const { dirname, normalize } = require('path')
 const { JS_BUNDLER_ZISI } = require('../../utils/consts')
 const { zipNodeJs } = require('../../zip_node')
 
-const { getSrcFilesAndExternalModules } = require('./src_files')
+const { getSrcFiles } = require('./src_files')
 const { getBasePath } = require('./utils/base_path')
 
 const zipZisi = async ({
@@ -21,12 +21,14 @@ const zipZisi = async ({
   srcPath,
   stat,
 }) => {
-  const { paths: srcFiles } = await getSrcFilesAndExternalModules({
+  const srcFiles = await getSrcFiles({
     bundler: JS_BUNDLER_ZISI,
+    config: {
+      ...config,
+      includedFilesBasePath: config.includedFilesBasePath || basePath,
+    },
     extension,
     featureFlags,
-    includedFiles: config.includedFiles,
-    includedFilesBasePath: config.includedFilesBasePath || basePath,
     mainFile,
     name,
     pluginsModulesPath,

src/zip.js

@@ -6,7 +6,7 @@ const pMap = require('p-map')
 const { getFlags } = require('./feature_flags')
 const { createManifest } = require('./manifest')
 const { getFunctionsFromPaths } = require('./runtimes')
-const { getPluginsModulesPath } = require('./runtimes/node/bundlers/zisi')
+const { getPluginsModulesPath } = require('./runtimes/node/utils/plugin_modules_path')
 const { ARCHIVE_FORMAT_NONE, ARCHIVE_FORMAT_ZIP } = require('./utils/consts')
 const { listFunctionsDirectories, resolveFunctionsDirectories } = require('./utils/fs')
 const { removeFalsy } = require('./utils/remove_falsy')