chore: Prepare for the switch of backend plugins to dynamic plugins (#…

…601)

* Add `/dynamic-plugins` to the monorepo

Signed-off-by: David Festal <dfestal@redhat.com>

* Allow duplicate plugin-specific app-config values
if they are equal.

Signed-off-by: David Festal <dfestal@redhat.com>

* Use a `dynamic-plugins.default.yaml` file in the container image

if nothing is specified in the Helm chart.

Signed-off-by: David Festal <dfestal@redhat.com>

* Securely support links in dynamic plugin archives

Signed-off-by: David Festal <dfestal@redhat.com>

* Support optional in the main index.ts file

* Skip .eslintrc.js files in lint-stage

Signed-off-by: David Festal <dfestal@redhat.com>

* Add changeset

Signed-off-by: David Festal <dfestal@redhat.com>

* Fox review comment

Signed-off-by: David Festal <dfestal@redhat.com>

* Fix new review comment

Signed-off-by: David Festal <dfestal@redhat.com>

---------

Signed-off-by: David Festal <dfestal@redhat.com>

.changeset/thin-frogs-jog.md

@@ -0,0 +1,6 @@
+---
+'backend': patch
+'app': patch
+---
+
+Prepare the showcase application for the switch of most plugins from static to dynamic loading.

.gitignore

@@ -52,3 +52,8 @@ site
 # Cypress
 **/cypress/downloads
 **/cypress/screenshots
+
+# Dynamic plugins root content
+dynamic-plugins-root/*
+!dynamic-plugins-root/.gitkeep
+dynamic-plugins/*/dist-dynamic/src

app-config.yaml

@@ -339,3 +339,6 @@ enabled:
   permission: ${PERMISSION_ENABLED}
   metrics: ${METRICS_ENABLED}
   aap: ${AAP_ENABLED}
+
+dynamicPlugins:
+  rootDirectory: dynamic-plugins-root

docker/Dockerfile

@@ -46,6 +46,7 @@ RUN chmod +x $YARN
 # Stage 2 - Install dependencies
 FROM skeleton AS deps
 
+COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins/ ./dynamic-plugins/
 COPY $EXTERNAL_SOURCE_NESTED/package.json $EXTERNAL_SOURCE_NESTED/yarn.lock ./
 COPY $EXTERNAL_SOURCE_NESTED/packages/app/package.json ./packages/app/package.json
 COPY $EXTERNAL_SOURCE_NESTED/packages/backend/package.json ./packages/backend/package.json
@@ -66,7 +67,18 @@ RUN rm app-config.yaml && mv app-config.example.yaml app-config.yaml
 RUN $YARN build --filter=backend
 
 # Build dynamic plugins
-RUN $YARN --cwd ./dynamic-plugins export-dynamic
+RUN $YARN export-dynamic
+RUN $YARN clean-dynamic-sources
+RUN mkdir -p dynamic-plugins-root && \
+    cd dynamic-plugins-root && \
+    rm -Rf * && \
+    for pkg in $CONTAINER_SOURCE/dynamic-plugins/*/dist-dynamic; do \
+      if [ -d $pkg ]; then \
+        archive=$(npm pack $pkg) && \
+        tar -xzf "$archive" && rm "$archive" && \
+        mv package $(echo $archive | sed -e 's:\.tgz$::'); \
+      fi; \
+    done
 
 # Stage 4 - Build the actual backend image and install production dependencies
 FROM skeleton AS cleanup
@@ -84,6 +96,7 @@ RUN tar xzf $TARBALL_PATH/skeleton.tar.gz; tar xzf $TARBALL_PATH/bundle.tar.gz;
 # Copy app-config files needed in runtime
 # Upstream only
 COPY $EXTERNAL_SOURCE_NESTED/app-config*.yaml ./
+COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins.default.yaml ./
 
 # Install production dependencies
 # hadolint ignore=DL3059
@@ -118,6 +131,10 @@ RUN chmod a+r ./install-dynamic-plugins.py
 COPY --from=build $CONTAINER_SOURCE/dynamic-plugins/ ./dynamic-plugins/
 RUN chmod -R a+r ./dynamic-plugins/
 
+# Copy default dynamic plugins root
+COPY --from=build $CONTAINER_SOURCE/dynamic-plugins-root/ ./dynamic-plugins-root/
+RUN chmod -R a+r ./dynamic-plugins-root/
+
 # The fix-permissions script is important when operating in environments that dynamically use a random UID at runtime, such as OpenShift.
 # The upstream backstage image does not account for this and it causes the container to fail at runtime.
 RUN fix-permissions ./

docker/brew.Dockerfile

@@ -46,6 +46,7 @@ COPY $EXTERNAL_SOURCE_NESTED/.yarnrc.yml ./
 RUN chmod +x $YARN
 
 # Stage 2 - Install dependencies
+COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins/ ./dynamic-plugins/
 COPY $EXTERNAL_SOURCE_NESTED/package.json $EXTERNAL_SOURCE_NESTED/yarn.lock ./
 COPY $EXTERNAL_SOURCE_NESTED/packages/app/package.json ./packages/app/package.json
 COPY $EXTERNAL_SOURCE_NESTED/packages/backend/package.json ./packages/backend/package.json
@@ -108,8 +109,18 @@ RUN git config --global --add safe.directory ./
 RUN $YARN build --filter=backend
 
 # Build dynamic plugins
-# hadolint ignore=DL3059
-RUN $YARN --cwd ./dynamic-plugins export-dynamic
+RUN $YARN export-dynamic
+RUN $YARN clean-dynamic-sources
+RUN mkdir -p dynamic-plugins-root && \
+    cd dynamic-plugins-root && \
+    rm -Rf * && \
+    for pkg in $CONTAINER_SOURCE/dynamic-plugins/*/dist-dynamic; do \
+      if [ -d $pkg ]; then \
+        archive=$(npm pack $pkg) && \
+        tar -xzf "$archive" && rm "$archive" && \
+        mv package $(echo $archive | sed -e 's:\.tgz$::'); \
+      fi; \
+    done
 
 # Stage 4 - Build the actual backend image and install production dependencies
 
@@ -161,7 +172,7 @@ RUN microdnf update -y && \
         pip3.11 install --user --no-cache-dir -r requirements.txt -r requirements-build.txt; \
     popd >/dev/null; \
     microdnf clean all; rm -fr $CONTAINER_SOURCE/upstream2
-    
+
 # Downstream only - copy from builder, not cleanup stage
 COPY --from=builder --chown=1001:1001 $CONTAINER_SOURCE/ ./
 
@@ -173,6 +184,10 @@ RUN chmod a+r ./install-dynamic-plugins.py
 COPY --from=builder $CONTAINER_SOURCE/dynamic-plugins/ ./dynamic-plugins/
 RUN chmod -R a+r ./dynamic-plugins/
 
+# Copy default dynamic plugins root
+COPY --from=build $CONTAINER_SOURCE/dynamic-plugins-root/ ./dynamic-plugins-root/
+RUN chmod -R a+r ./dynamic-plugins-root/
+
 # The fix-permissions script is important when operating in environments that dynamically use a random UID at runtime, such as OpenShift.
 # The upstream backstage image does not account for this and it causes the container to fail at runtime.
 RUN fix-permissions ./

docker/install-dynamic-plugins.py

@@ -1,24 +1,61 @@
+#
+# Copyright (c) 2023 Red Hat, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import os
 import sys
 import yaml
 import tarfile
 import shutil
 import subprocess
 
+# This script is used to install dynamic plugins in the Backstage application,
+# and is available in the container image to be called at container initialization,
+# for example in an init container when using Kubernetes.
+#
+# It expects, as the only argument, the path to the root directory where
+# the dynamic plugins will be installed.
+#
+# Additionally The MAX_ENTRY_SIZE environment variable can be defined to set
+# the maximum size of a file in the archive (default: 10MB).
+#
+# It expects the `dynamic-plugins.yaml` file to be present in the current directory and
+# to contain the list of plugins to install along with their optional configuration.
+#
+# The `dynamic-plugins.yaml` file must be a list of objects with the following properties:
+#   - `package`: the NPM package to install (either a package name or a path to a local package)
+#   - `pluginConfig`: an optional plugin-specific configuration fragment
+#
+# For each package mentioned in the `dynamic-plugins.yaml` file, the script will:
+#   - call `npm pack` to get the package archive and extract it in the dynamic plugins root directory
+#   - merge the plugin-specific configuration fragment in a global configuration file named `app-config.dynamic-plugins.yaml`
+#
+
 class InstallException(Exception):
     """Exception class from which every exception in this library will derive."""
     pass
 
-def merge(source, destination):
+def merge(source, destination, prefix = ''):
     for key, value in source.items():
         if isinstance(value, dict):
             # get node or create one
             node = destination.setdefault(key, {})
-            merge(value, node)
+            merge(value, node, key + '.')
         else:
             # if key exists in destination trigger an error
-            if key in destination:
-                raise InstallException('Config key ' + key + ' defined for 2 dynamic plugins')
+            if key in destination and destination[key] != value:
+                raise InstallException("Config key '" + prefix + key + "' defined differently for 2 dynamic plugins")
 
             destination[key] = value
 
@@ -28,16 +65,20 @@ def main():
     dynamicPluginsRoot = sys.argv[1]
     maxEntrySize = int(os.environ.get('MAX_ENTRY_SIZE', 10000000))
 
-    dynamicPluginsFile = os.path.join(dynamicPluginsRoot, 'dynamic-plugins.yaml')
+    dynamicPluginsFile = 'dynamic-plugins.yaml'
+    dynamicPluginsDefaultFile = 'dynamic-plugins.default.yaml'
     dynamicPluginsGlobalConfigFile = os.path.join(dynamicPluginsRoot, 'app-config.dynamic-plugins.yaml')
 
     # test if file dynamic-plugins.yaml exists
     if not os.path.isfile(dynamicPluginsFile):
-        print(f'No {dynamicPluginsFile} file found. Skipping dynamic plugins installation.')
-        with open(dynamicPluginsGlobalConfigFile, 'w') as file:
-            file.write('')
-            file.close()
-        exit(0)
+        print(f'No {dynamicPluginsFile} file found, trying {dynamicPluginsDefaultFile} file.')
+        dynamicPluginsFile = dynamicPluginsDefaultFile
+        if not os.path.isfile(dynamicPluginsFile):
+            print(f'No {dynamicPluginsFile} file found. Skipping dynamic plugins installation.')
+            with open(dynamicPluginsGlobalConfigFile, 'w') as file:
+                file.write('')
+                file.close()
+            exit(0)
 
     with open(dynamicPluginsFile, 'r') as file:
         plugins = yaml.safe_load(file)
@@ -74,6 +115,7 @@ def main():
 
         archive = os.path.join(dynamicPluginsRoot, completed.stdout.decode('utf-8').strip())
         directory = archive.replace('.tgz', '')
+        directoryRealpath = os.path.realpath(directory)
 
         print('\t==> Removing previous plugin directory', directory, flush=True)
         shutil.rmtree(directory, ignore_errors=True, onerror=None)
@@ -90,13 +132,33 @@ def main():
                 if member.size > maxEntrySize:
                     raise InstallException('Zip bomb detected in ' + member.name)
 
-                # Remove the `package/` prefix from the file name
-                member.name = member.name[8:]
+                member.name = member.name.removeprefix('package/')
                 file.extract(member, path=directory)
             elif member.isdir():
                 print('\t\tSkipping directory entry', member.name, flush=True)
+            elif member.islnk() or member.issym():
+                if not member.linkpath.startswith('package/'):
+                  raise InstallException('NPM package archive contains a link outside of the archive: ' + member.name + ' -> ' + member.linkpath)
+
+                member.name = member.name.removeprefix('package/')
+                member.linkpath = member.linkpath.removeprefix('package/')
+
+                realpath = os.path.realpath(os.path.join(directory, *os.path.split(member.linkname)))
+                if not realpath.startswith(directoryRealpath):
+                  raise InstallException('NPM package archive contains a link outside of the archive: ' + member.name + ' -> ' + member.linkpath)
+
+                file.extract(member, path=directory)
             else:
-                raise InstallException('NPM package archive contains a non regular file: ' + member.name)
+              if member.type == tarfile.CHRTYPE:
+                  type_str = "character device"
+              elif member.type == tarfile.BLKTYPE:
+                  type_str = "block device"
+              elif member.type == tarfile.FIFOTYPE:
+                  type_str = "FIFO"
+              else:
+                  type_str = "unknown"
+
+              raise InstallException('NPM package archive contains a non regular file: ' + member.name + ' - ' + type_str)
 
         file.close()
 

dynamic-plugins.default.yaml

@@ -0,0 +1 @@
+- package: ./dynamic-plugins/scaffolder-backend-module-utils-wrapped/dist-dynamic

package.json

@@ -12,6 +12,8 @@
     "start-backend": "turbo run start --filter=backend",
     "build": "turbo run build",
     "tsc": "tsc",
+    "export-dynamic": "turbo run export-dynamic --concurrency 1",
+    "clean-dynamic-sources": "turbo run clean-dynamic-sources",
     "clean": "turbo run clean",
     "test": "turbo run test",
     "test:e2e": "turbo run test:e2e",
@@ -29,7 +31,8 @@
   "workspaces": {
     "packages": [
       "packages/*",
-      "plugins/*"
+      "plugins/*",
+      "dynamic-plugins/*"
     ]
   },
   "devDependencies": {
@@ -51,7 +54,8 @@
   "prettier": "@spotify/prettier-config",
   "lint-staged": {
     "*": "yarn run prettier:fix",
-    "*.{js,jsx,ts,tsx,mjs,cjs}": "yarn run lint -- -- --fix"
+    "*.{jsx,ts,tsx,mjs,cjs}": "yarn run lint -- -- --fix",
+    "!(.eslintrc).js": "yarn run lint -- -- --fix"
   },
   "packageManager": "yarn@1.22.19"
 }

packages/backend/src/index.ts

@@ -113,12 +113,28 @@ type AddPlugin = {
   isOptional?: false;
 } & AddPluginBase;
 
+type OptionalPluginOptions = {
+  key?: string;
+  path?: string;
+};
+
 type AddOptionalPlugin = {
   isOptional: true;
   config: Config;
-  options?: { key?: string; path?: string };
+  options?: OptionalPluginOptions;
 } & AddPluginBase;
 
+const OPTIONAL_DYNAMIC_PLUGINS: { [key: string]: OptionalPluginOptions } = {
+  techdocs: {},
+  argocd: {},
+  sonarqube: {},
+  kubernetes: {},
+  'azure-devops': { key: 'enabled.azureDevOps' },
+  jenkins: {},
+  ocm: {},
+  gitlab: {},
+} as const satisfies { [key: string]: OptionalPluginOptions };
+
 async function addPlugin(args: AddPlugin | AddOptionalPlugin): Promise<void> {
   const { isOptional, plugin, apiRouter, createEnv, router, options } = args;
 
@@ -132,10 +148,13 @@ async function addPlugin(args: AddPlugin | AddOptionalPlugin): Promise<void> {
     );
     apiRouter.use(options?.path ?? `/${plugin}`, await router(pluginEnv));
     console.log(`Using backend plugin ${plugin}...`);
+  } else if (isOptional) {
+    console.log(`Backend plugin ${plugin} is disabled`);
   }
 }
 
 type AddRouterBase = {
+  isOptional?: boolean;
   name: string;
   service: ServiceBuilder;
   root: string;
@@ -284,13 +303,21 @@ async function main() {
     if (plugin.installer.kind === 'legacy') {
       const pluginRouter = plugin.installer.router;
       if (pluginRouter !== undefined) {
-        const pluginEnv = useHotMemoize(module, () =>
-          createEnv(pluginRouter.pluginID),
-        );
-        apiRouter.use(
-          `/${pluginRouter.pluginID}`,
-          await pluginRouter.createPlugin(pluginEnv),
-        );
+        let optionals = {};
+        if (pluginRouter.pluginID in OPTIONAL_DYNAMIC_PLUGINS) {
+          optionals = {
+            isOptional: true,
+            config: config,
+            options: OPTIONAL_DYNAMIC_PLUGINS[pluginRouter.pluginID],
+          };
+        }
+        await addPlugin({
+          plugin: pluginRouter.pluginID,
+          apiRouter,
+          createEnv,
+          router: pluginRouter.createPlugin,
+          ...optionals,
+        });
       }
     }
   }

tsconfig.json

@@ -4,7 +4,8 @@
     "packages/*/src",
     "plugins/*/src",
     "plugins/*/dev",
-    "plugins/*/migrations"
+    "plugins/*/migrations",
+    "dynamic-plugins/*/src"
   ],
   "exclude": ["node_modules"],
   "compilerOptions": {