diff --git a/.changeset/thin-frogs-jog.md b/.changeset/thin-frogs-jog.md new file mode 100644 index 0000000000..10a1e7fbfb --- /dev/null +++ b/.changeset/thin-frogs-jog.md @@ -0,0 +1,6 @@ +--- +'backend': patch +'app': patch +--- + +Prepare the showcase application for the switch of most plugins from static to dynamic loading. diff --git a/.gitignore b/.gitignore index 5f5f77e675..27026a32f0 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,8 @@ site # Cypress **/cypress/downloads **/cypress/screenshots + +# Dynamic plugins root content +dynamic-plugins-root/* +!dynamic-plugins-root/.gitkeep +dynamic-plugins/*/dist-dynamic/src diff --git a/app-config.yaml b/app-config.yaml index 79b5bce5e2..b70480c04c 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -339,3 +339,6 @@ enabled: permission: ${PERMISSION_ENABLED} metrics: ${METRICS_ENABLED} aap: ${AAP_ENABLED} + +dynamicPlugins: + rootDirectory: dynamic-plugins-root diff --git a/docker/Dockerfile b/docker/Dockerfile index 844baf9982..42727ee1d8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -46,6 +46,7 @@ RUN chmod +x $YARN # Stage 2 - Install dependencies FROM skeleton AS deps +COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins/ ./dynamic-plugins/ COPY $EXTERNAL_SOURCE_NESTED/package.json $EXTERNAL_SOURCE_NESTED/yarn.lock ./ COPY $EXTERNAL_SOURCE_NESTED/packages/app/package.json ./packages/app/package.json COPY $EXTERNAL_SOURCE_NESTED/packages/backend/package.json ./packages/backend/package.json @@ -66,7 +67,18 @@ RUN rm app-config.yaml && mv app-config.example.yaml app-config.yaml RUN $YARN build --filter=backend # Build dynamic plugins -RUN $YARN --cwd ./dynamic-plugins export-dynamic +RUN $YARN export-dynamic +RUN $YARN clean-dynamic-sources +RUN mkdir -p dynamic-plugins-root && \ + cd dynamic-plugins-root && \ + rm -Rf * && \ + for pkg in $CONTAINER_SOURCE/dynamic-plugins/*/dist-dynamic; do \ + if [ -d $pkg ]; then \ + archive=$(npm pack $pkg) && \ + tar -xzf "$archive" && rm "$archive" && \ + mv package $(echo $archive | sed -e 's:\.tgz$::'); \ + fi; \ + done # Stage 4 - Build the actual backend image and install production dependencies FROM skeleton AS cleanup @@ -84,6 +96,7 @@ RUN tar xzf $TARBALL_PATH/skeleton.tar.gz; tar xzf $TARBALL_PATH/bundle.tar.gz; # Copy app-config files needed in runtime # Upstream only COPY $EXTERNAL_SOURCE_NESTED/app-config*.yaml ./ +COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins.default.yaml ./ # Install production dependencies # hadolint ignore=DL3059 @@ -118,6 +131,10 @@ RUN chmod a+r ./install-dynamic-plugins.py COPY --from=build $CONTAINER_SOURCE/dynamic-plugins/ ./dynamic-plugins/ RUN chmod -R a+r ./dynamic-plugins/ +# Copy default dynamic plugins root +COPY --from=build $CONTAINER_SOURCE/dynamic-plugins-root/ ./dynamic-plugins-root/ +RUN chmod -R a+r ./dynamic-plugins-root/ + # The fix-permissions script is important when operating in environments that dynamically use a random UID at runtime, such as OpenShift. # The upstream backstage image does not account for this and it causes the container to fail at runtime. RUN fix-permissions ./ diff --git a/docker/brew.Dockerfile b/docker/brew.Dockerfile index f007a01da7..6be48cecd7 100644 --- a/docker/brew.Dockerfile +++ b/docker/brew.Dockerfile @@ -46,6 +46,7 @@ COPY $EXTERNAL_SOURCE_NESTED/.yarnrc.yml ./ RUN chmod +x $YARN # Stage 2 - Install dependencies +COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins/ ./dynamic-plugins/ COPY $EXTERNAL_SOURCE_NESTED/package.json $EXTERNAL_SOURCE_NESTED/yarn.lock ./ COPY $EXTERNAL_SOURCE_NESTED/packages/app/package.json ./packages/app/package.json COPY $EXTERNAL_SOURCE_NESTED/packages/backend/package.json ./packages/backend/package.json @@ -108,8 +109,18 @@ RUN git config --global --add safe.directory ./ RUN $YARN build --filter=backend # Build dynamic plugins -# hadolint ignore=DL3059 -RUN $YARN --cwd ./dynamic-plugins export-dynamic +RUN $YARN export-dynamic +RUN $YARN clean-dynamic-sources +RUN mkdir -p dynamic-plugins-root && \ + cd dynamic-plugins-root && \ + rm -Rf * && \ + for pkg in $CONTAINER_SOURCE/dynamic-plugins/*/dist-dynamic; do \ + if [ -d $pkg ]; then \ + archive=$(npm pack $pkg) && \ + tar -xzf "$archive" && rm "$archive" && \ + mv package $(echo $archive | sed -e 's:\.tgz$::'); \ + fi; \ + done # Stage 4 - Build the actual backend image and install production dependencies @@ -161,7 +172,7 @@ RUN microdnf update -y && \ pip3.11 install --user --no-cache-dir -r requirements.txt -r requirements-build.txt; \ popd >/dev/null; \ microdnf clean all; rm -fr $CONTAINER_SOURCE/upstream2 - + # Downstream only - copy from builder, not cleanup stage COPY --from=builder --chown=1001:1001 $CONTAINER_SOURCE/ ./ @@ -173,6 +184,10 @@ RUN chmod a+r ./install-dynamic-plugins.py COPY --from=builder $CONTAINER_SOURCE/dynamic-plugins/ ./dynamic-plugins/ RUN chmod -R a+r ./dynamic-plugins/ +# Copy default dynamic plugins root +COPY --from=build $CONTAINER_SOURCE/dynamic-plugins-root/ ./dynamic-plugins-root/ +RUN chmod -R a+r ./dynamic-plugins-root/ + # The fix-permissions script is important when operating in environments that dynamically use a random UID at runtime, such as OpenShift. # The upstream backstage image does not account for this and it causes the container to fail at runtime. RUN fix-permissions ./ diff --git a/docker/install-dynamic-plugins.py b/docker/install-dynamic-plugins.py index 99989659ad..7fbb071347 100644 --- a/docker/install-dynamic-plugins.py +++ b/docker/install-dynamic-plugins.py @@ -1,3 +1,18 @@ +# +# Copyright (c) 2023 Red Hat, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + import os import sys import yaml @@ -5,20 +20,42 @@ import shutil import subprocess +# This script is used to install dynamic plugins in the Backstage application, +# and is available in the container image to be called at container initialization, +# for example in an init container when using Kubernetes. +# +# It expects, as the only argument, the path to the root directory where +# the dynamic plugins will be installed. +# +# Additionally The MAX_ENTRY_SIZE environment variable can be defined to set +# the maximum size of a file in the archive (default: 10MB). +# +# It expects the `dynamic-plugins.yaml` file to be present in the current directory and +# to contain the list of plugins to install along with their optional configuration. +# +# The `dynamic-plugins.yaml` file must be a list of objects with the following properties: +# - `package`: the NPM package to install (either a package name or a path to a local package) +# - `pluginConfig`: an optional plugin-specific configuration fragment +# +# For each package mentioned in the `dynamic-plugins.yaml` file, the script will: +# - call `npm pack` to get the package archive and extract it in the dynamic plugins root directory +# - merge the plugin-specific configuration fragment in a global configuration file named `app-config.dynamic-plugins.yaml` +# + class InstallException(Exception): """Exception class from which every exception in this library will derive.""" pass -def merge(source, destination): +def merge(source, destination, prefix = ''): for key, value in source.items(): if isinstance(value, dict): # get node or create one node = destination.setdefault(key, {}) - merge(value, node) + merge(value, node, key + '.') else: # if key exists in destination trigger an error - if key in destination: - raise InstallException('Config key ' + key + ' defined for 2 dynamic plugins') + if key in destination and destination[key] != value: + raise InstallException("Config key '" + prefix + key + "' defined differently for 2 dynamic plugins") destination[key] = value @@ -28,16 +65,20 @@ def main(): dynamicPluginsRoot = sys.argv[1] maxEntrySize = int(os.environ.get('MAX_ENTRY_SIZE', 10000000)) - dynamicPluginsFile = os.path.join(dynamicPluginsRoot, 'dynamic-plugins.yaml') + dynamicPluginsFile = 'dynamic-plugins.yaml' + dynamicPluginsDefaultFile = 'dynamic-plugins.default.yaml' dynamicPluginsGlobalConfigFile = os.path.join(dynamicPluginsRoot, 'app-config.dynamic-plugins.yaml') # test if file dynamic-plugins.yaml exists if not os.path.isfile(dynamicPluginsFile): - print(f'No {dynamicPluginsFile} file found. Skipping dynamic plugins installation.') - with open(dynamicPluginsGlobalConfigFile, 'w') as file: - file.write('') - file.close() - exit(0) + print(f'No {dynamicPluginsFile} file found, trying {dynamicPluginsDefaultFile} file.') + dynamicPluginsFile = dynamicPluginsDefaultFile + if not os.path.isfile(dynamicPluginsFile): + print(f'No {dynamicPluginsFile} file found. Skipping dynamic plugins installation.') + with open(dynamicPluginsGlobalConfigFile, 'w') as file: + file.write('') + file.close() + exit(0) with open(dynamicPluginsFile, 'r') as file: plugins = yaml.safe_load(file) @@ -74,6 +115,7 @@ def main(): archive = os.path.join(dynamicPluginsRoot, completed.stdout.decode('utf-8').strip()) directory = archive.replace('.tgz', '') + directoryRealpath = os.path.realpath(directory) print('\t==> Removing previous plugin directory', directory, flush=True) shutil.rmtree(directory, ignore_errors=True, onerror=None) @@ -90,13 +132,33 @@ def main(): if member.size > maxEntrySize: raise InstallException('Zip bomb detected in ' + member.name) - # Remove the `package/` prefix from the file name - member.name = member.name[8:] + member.name = member.name.removeprefix('package/') file.extract(member, path=directory) elif member.isdir(): print('\t\tSkipping directory entry', member.name, flush=True) + elif member.islnk() or member.issym(): + if not member.linkpath.startswith('package/'): + raise InstallException('NPM package archive contains a link outside of the archive: ' + member.name + ' -> ' + member.linkpath) + + member.name = member.name.removeprefix('package/') + member.linkpath = member.linkpath.removeprefix('package/') + + realpath = os.path.realpath(os.path.join(directory, *os.path.split(member.linkname))) + if not realpath.startswith(directoryRealpath): + raise InstallException('NPM package archive contains a link outside of the archive: ' + member.name + ' -> ' + member.linkpath) + + file.extract(member, path=directory) else: - raise InstallException('NPM package archive contains a non regular file: ' + member.name) + if member.type == tarfile.CHRTYPE: + type_str = "character device" + elif member.type == tarfile.BLKTYPE: + type_str = "block device" + elif member.type == tarfile.FIFOTYPE: + type_str = "FIFO" + else: + type_str = "unknown" + + raise InstallException('NPM package archive contains a non regular file: ' + member.name + ' - ' + type_str) file.close() diff --git a/dynamic-plugins-root/.gitkeep b/dynamic-plugins-root/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dynamic-plugins.default.yaml b/dynamic-plugins.default.yaml new file mode 100644 index 0000000000..a6c2e91e1b --- /dev/null +++ b/dynamic-plugins.default.yaml @@ -0,0 +1 @@ +- package: ./dynamic-plugins/scaffolder-backend-module-utils-wrapped/dist-dynamic diff --git a/dynamic-plugins/.eslintignore b/dynamic-plugins/.eslintignore deleted file mode 100644 index 2d586d76da..0000000000 --- a/dynamic-plugins/.eslintignore +++ /dev/null @@ -1,4 +0,0 @@ -**/node_modules/** -**/dist/** -**/dist-dynamic/** -**/templates/** diff --git a/dynamic-plugins/.eslintrc b/dynamic-plugins/.eslintrc deleted file mode 100644 index 8efdd5f068..0000000000 --- a/dynamic-plugins/.eslintrc +++ /dev/null @@ -1,6 +0,0 @@ -{ - "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"], - "parser": "@typescript-eslint/parser", - "plugins": ["@typescript-eslint"], - "root": true -} diff --git a/dynamic-plugins/.gitkeep b/dynamic-plugins/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dynamic-plugins/package.json b/dynamic-plugins/package.json deleted file mode 100644 index 87650cb8c2..0000000000 --- a/dynamic-plugins/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "private": true, - "scripts": { - "export-dynamic": "for D in $(ls -d */ 2>/dev/null); do yarn --cwd \"${D}\" install --frozen-lockfile && yarn --cwd \"${D}\" export-dynamic && rm -Rf ${D}/node_modules ${D}/.yarn ${D}/dist; done" - } -} diff --git a/package.json b/package.json index 633bd7c8c4..5bc4cfb054 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,8 @@ "start-backend": "turbo run start --filter=backend", "build": "turbo run build", "tsc": "tsc", + "export-dynamic": "turbo run export-dynamic --concurrency 1", + "clean-dynamic-sources": "turbo run clean-dynamic-sources", "clean": "turbo run clean", "test": "turbo run test", "test:e2e": "turbo run test:e2e", @@ -29,7 +31,8 @@ "workspaces": { "packages": [ "packages/*", - "plugins/*" + "plugins/*", + "dynamic-plugins/*" ] }, "devDependencies": { @@ -51,7 +54,8 @@ "prettier": "@spotify/prettier-config", "lint-staged": { "*": "yarn run prettier:fix", - "*.{js,jsx,ts,tsx,mjs,cjs}": "yarn run lint -- -- --fix" + "*.{jsx,ts,tsx,mjs,cjs}": "yarn run lint -- -- --fix", + "!(.eslintrc).js": "yarn run lint -- -- --fix" }, "packageManager": "yarn@1.22.19" } diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 5703ff1401..7e8d404f49 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -113,12 +113,28 @@ type AddPlugin = { isOptional?: false; } & AddPluginBase; +type OptionalPluginOptions = { + key?: string; + path?: string; +}; + type AddOptionalPlugin = { isOptional: true; config: Config; - options?: { key?: string; path?: string }; + options?: OptionalPluginOptions; } & AddPluginBase; +const OPTIONAL_DYNAMIC_PLUGINS: { [key: string]: OptionalPluginOptions } = { + techdocs: {}, + argocd: {}, + sonarqube: {}, + kubernetes: {}, + 'azure-devops': { key: 'enabled.azureDevOps' }, + jenkins: {}, + ocm: {}, + gitlab: {}, +} as const satisfies { [key: string]: OptionalPluginOptions }; + async function addPlugin(args: AddPlugin | AddOptionalPlugin): Promise { const { isOptional, plugin, apiRouter, createEnv, router, options } = args; @@ -132,10 +148,13 @@ async function addPlugin(args: AddPlugin | AddOptionalPlugin): Promise { ); apiRouter.use(options?.path ?? `/${plugin}`, await router(pluginEnv)); console.log(`Using backend plugin ${plugin}...`); + } else if (isOptional) { + console.log(`Backend plugin ${plugin} is disabled`); } } type AddRouterBase = { + isOptional?: boolean; name: string; service: ServiceBuilder; root: string; @@ -284,13 +303,21 @@ async function main() { if (plugin.installer.kind === 'legacy') { const pluginRouter = plugin.installer.router; if (pluginRouter !== undefined) { - const pluginEnv = useHotMemoize(module, () => - createEnv(pluginRouter.pluginID), - ); - apiRouter.use( - `/${pluginRouter.pluginID}`, - await pluginRouter.createPlugin(pluginEnv), - ); + let optionals = {}; + if (pluginRouter.pluginID in OPTIONAL_DYNAMIC_PLUGINS) { + optionals = { + isOptional: true, + config: config, + options: OPTIONAL_DYNAMIC_PLUGINS[pluginRouter.pluginID], + }; + } + await addPlugin({ + plugin: pluginRouter.pluginID, + apiRouter, + createEnv, + router: pluginRouter.createPlugin, + ...optionals, + }); } } } diff --git a/tsconfig.json b/tsconfig.json index aebce0ce34..0601794857 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,8 @@ "packages/*/src", "plugins/*/src", "plugins/*/dev", - "plugins/*/migrations" + "plugins/*/migrations", + "dynamic-plugins/*/src" ], "exclude": ["node_modules"], "compilerOptions": { diff --git a/turbo.json b/turbo.json index 026bd779e6..1dfb7933d3 100644 --- a/turbo.json +++ b/turbo.json @@ -1,6 +1,12 @@ { "$schema": "https://turbo.build/schema.json", "pipeline": { + "export-dynamic": { + "outputs": ["dist-dynamic/**"] + }, + "clean-dynamic-sources": { + "cache": false + }, "start": { "cache": false, "persistent": true