From 06b8bad39e8f92f9f348a751076fc016b6583a79 Mon Sep 17 00:00:00 2001 From: Daniel Lee Date: Wed, 25 Jan 2023 13:13:30 -0800 Subject: [PATCH] Add python support for Functions Emulator. (#5423) Add support for loading and serving functions written using the Firebase Functions Python SDK (WIP: https://github.com/firebase/firebase-functions-python) This PR is a fork of https://github.com/firebase/firebase-tools/pull/4653 extended with support for emulating Python functions. Python Runtime Delegate implementation is unsurprising but does include some additional wrapper code to make sure all commands (e.g. spinning up the admin server) runs within the virtualenv environment. For now, we hardcode virtual environment `venv` directory exists on the developer's laptop, but we'll later add support for specifying arbitrary directory for specifying virtualenv directory via firebase.json configuration. Another note is that each emulated Python function will bind to a Port instead of Unix Domain Socket (UDS) as done when emulating Node.js function. This is because there is no straightfoward, platform-neutral way to bind python webserver to UDS. Finding large number of open port might have a bit more performance penalty and cause bugs due to race condition (similar to https://github.com/firebase/firebase-tools/issues/5418) but it seems that we have no other choice atm. --- .../emulator-tests/functionsEmulator.spec.ts | 1 + .../functions/runtimes/discovery/index.ts | 2 +- src/deploy/functions/runtimes/index.ts | 7 +- src/deploy/functions/runtimes/python/index.ts | 152 ++++++++++++++++++ src/emulator/functionsEmulator.ts | 124 +++++++++++--- src/emulator/functionsRuntimeWorker.ts | 32 ++-- src/functions/python.ts | 32 ++++ .../runtimes/discovery/index.spec.ts | 4 +- .../functions/runtimes/python/index.spec.ts | 45 ++++++ .../emulators/functionsRuntimeWorker.spec.ts | 4 +- 10 files changed, 354 insertions(+), 49 deletions(-) create mode 100644 src/deploy/functions/runtimes/python/index.ts create mode 100644 src/functions/python.ts create mode 100644 src/test/deploy/functions/runtimes/python/index.spec.ts diff --git a/scripts/emulator-tests/functionsEmulator.spec.ts b/scripts/emulator-tests/functionsEmulator.spec.ts index 30d336bc0b4..f6591d0e8e3 100644 --- a/scripts/emulator-tests/functionsEmulator.spec.ts +++ b/scripts/emulator-tests/functionsEmulator.spec.ts @@ -38,6 +38,7 @@ const TEST_BACKEND = { secretEnv: [], codebase: "default", bin: process.execPath, + runtime: "nodejs14", // NOTE: Use the following node bin path if you want to run test cases directly from your IDE. // bin: path.join(MODULE_ROOT, "node_modules/.bin/ts-node"), }; diff --git a/src/deploy/functions/runtimes/discovery/index.ts b/src/deploy/functions/runtimes/discovery/index.ts index 809671fa93a..e5e0e8ff71e 100644 --- a/src/deploy/functions/runtimes/discovery/index.ts +++ b/src/deploy/functions/runtimes/discovery/index.ts @@ -72,7 +72,7 @@ export async function detectFromPort( while (true) { try { - res = await Promise.race([fetch(`http://localhost:${port}/__/functions.yaml`), timedOut]); + res = await Promise.race([fetch(`http://127.0.0.1:${port}/__/functions.yaml`), timedOut]); break; } catch (err: any) { // Allow us to wait until the server is listening. diff --git a/src/deploy/functions/runtimes/index.ts b/src/deploy/functions/runtimes/index.ts index ce327dd4f8c..8864b127f44 100644 --- a/src/deploy/functions/runtimes/index.ts +++ b/src/deploy/functions/runtimes/index.ts @@ -1,6 +1,7 @@ import * as backend from "../backend"; import * as build from "../build"; import * as node from "./node"; +import * as python from "./python"; import * as validate from "../validate"; import { FirebaseError } from "../../../error"; @@ -9,7 +10,7 @@ const RUNTIMES: string[] = ["nodejs10", "nodejs12", "nodejs14", "nodejs16", "nod // Experimental runtimes are part of the Runtime type, but are in a // different list to help guard against some day accidentally iterating over // and printing a hidden runtime to the user. -const EXPERIMENTAL_RUNTIMES: string[] = []; +const EXPERIMENTAL_RUNTIMES: string[] = ["python310", "python311"]; export type Runtime = typeof RUNTIMES[number] | typeof EXPERIMENTAL_RUNTIMES[number]; /** Runtimes that can be found in existing backends but not used for new functions. */ @@ -34,6 +35,8 @@ const MESSAGE_FRIENDLY_RUNTIMES: Record = { nodejs14: "Node.js 14", nodejs16: "Node.js 16", nodejs18: "Node.js 18", + python310: "Python 3.10", + python311: "Python 3.11 (Preview)", }; /** @@ -113,7 +116,7 @@ export interface DelegateContext { } type Factory = (context: DelegateContext) => Promise; -const factories: Factory[] = [node.tryCreateDelegate]; +const factories: Factory[] = [node.tryCreateDelegate, python.tryCreateDelegate]; /** * diff --git a/src/deploy/functions/runtimes/python/index.ts b/src/deploy/functions/runtimes/python/index.ts new file mode 100644 index 00000000000..4e4989f9df5 --- /dev/null +++ b/src/deploy/functions/runtimes/python/index.ts @@ -0,0 +1,152 @@ +import * as fs from "fs"; +import * as path from "path"; +import fetch from "node-fetch"; +import { promisify } from "util"; + +import * as portfinder from "portfinder"; + +import * as runtimes from ".."; +import * as backend from "../../backend"; +import * as discovery from "../discovery"; +import { logger } from "../../../../logger"; +import { runWithVirtualEnv } from "../../../../functions/python"; +import { FirebaseError } from "../../../../error"; +import { Build } from "../../build"; + +const LATEST_VERSION: runtimes.Runtime = "python310"; + +/** + * Create a runtime delegate for the Python runtime, if applicable. + * + * @param context runtimes.DelegateContext + * @return Delegate Python runtime delegate + */ +export async function tryCreateDelegate( + context: runtimes.DelegateContext +): Promise { + const requirementsTextPath = path.join(context.sourceDir, "requirements.txt"); + + if (!(await promisify(fs.exists)(requirementsTextPath))) { + logger.debug("Customer code is not Python code."); + return; + } + const runtime = context.runtime ? context.runtime : LATEST_VERSION; + if (!runtimes.isValidRuntime(runtime)) { + throw new FirebaseError(`Runtime ${runtime} is not a valid Python runtime`); + } + return Promise.resolve(new Delegate(context.projectId, context.sourceDir, runtime)); +} + +export class Delegate implements runtimes.RuntimeDelegate { + public readonly name = "python"; + constructor( + private readonly projectId: string, + private readonly sourceDir: string, + public readonly runtime: runtimes.Runtime + ) {} + + private _bin = ""; + private _modulesDir = ""; + + get bin(): string { + if (this._bin === "") { + this._bin = this.getPythonBinary(); + } + return this._bin; + } + + async modulesDir(): Promise { + if (!this._modulesDir) { + const child = runWithVirtualEnv( + [ + this.bin, + "-c", + '"import firebase_functions; import os; print(os.path.dirname(firebase_functions.__file__))"', + ], + this.sourceDir, + {} + ); + let out = ""; + child.stdout?.on("data", (chunk: Buffer) => { + const chunkString = chunk.toString(); + out = out + chunkString; + logger.debug(`stdout: ${chunkString}`); + }); + await new Promise((resolve, reject) => { + child.on("exit", resolve); + child.on("error", reject); + }); + this._modulesDir = out.trim(); + } + return this._modulesDir; + } + + getPythonBinary(): string { + if (process.platform === "win32") { + // There is no easy way to get specific version of python executable in Windows. + return "python.exe"; + } + if (this.runtime === "python310") { + return "python3.10"; + } else if (this.runtime === "python311") { + return "python3.11"; + } + return "python"; + } + + validate(): Promise { + // TODO: make sure firebase-functions is included as a dep + return Promise.resolve(); + } + + watch(): Promise<() => Promise> { + return Promise.resolve(() => Promise.resolve()); + } + + async build(): Promise { + return Promise.resolve(); + } + + async serveAdmin(port: number, envs: backend.EnvironmentVariables): Promise<() => Promise> { + const modulesDir = await this.modulesDir(); + const envWithAdminPort = { + ...envs, + ADMIN_PORT: port.toString(), + }; + const args = [this.bin, path.join(modulesDir, "private", "serving.py")]; + logger.debug( + `Running admin server with args: ${JSON.stringify(args)} and env: ${JSON.stringify( + envWithAdminPort + )} in ${this.sourceDir}` + ); + const childProcess = runWithVirtualEnv(args, this.sourceDir, envWithAdminPort); + return Promise.resolve(async () => { + await fetch(`http://127.0.0.1:${port}/__/quitquitquit`); + const quitTimeout = setTimeout(() => { + if (!childProcess.killed) { + childProcess.kill("SIGKILL"); + } + }, 10_000); + clearTimeout(quitTimeout); + }); + } + + async discoverBuild( + _configValues: backend.RuntimeConfigValues, + envs: backend.EnvironmentVariables + ): Promise { + let discovered = await discovery.detectFromYaml(this.sourceDir, this.projectId, this.runtime); + if (!discovered) { + const adminPort = await portfinder.getPortPromise({ + port: 8081, + }); + const killProcess = await this.serveAdmin(adminPort, envs); + try { + discovered = await discovery.detectFromPort(adminPort, this.projectId, this.runtime); + } finally { + await killProcess(); + } + } + return discovered; + } +} diff --git a/src/emulator/functionsEmulator.ts b/src/emulator/functionsEmulator.ts index d8124797e19..34722120d5e 100644 --- a/src/emulator/functionsEmulator.ts +++ b/src/emulator/functionsEmulator.ts @@ -15,6 +15,7 @@ import { track, trackEmulator } from "../track"; import { Constants } from "./constants"; import { EmulatorInfo, EmulatorInstance, Emulators, FunctionsExecutionMode } from "./types"; import * as chokidar from "chokidar"; +import * as portfinder from "portfinder"; import * as spawn from "cross-spawn"; import { ChildProcess } from "child_process"; @@ -43,7 +44,7 @@ import { RuntimeWorker, RuntimeWorkerPool } from "./functionsRuntimeWorker"; import { PubsubEmulator } from "./pubsubEmulator"; import { FirebaseError } from "../error"; import { WorkQueue, Work } from "./workQueue"; -import { allSettled, connectableHostname, createDestroyer, debounce } from "../utils"; +import { allSettled, connectableHostname, createDestroyer, debounce, randomInt } from "../utils"; import { getCredentialPathAsync } from "../defaultCredentials"; import { AdminSdkConfig, @@ -60,6 +61,7 @@ import { AUTH_BLOCKING_EVENTS, BEFORE_CREATE_EVENT } from "../functions/events/v import { BlockingFunctionsConfig } from "../gcp/identityPlatform"; import { resolveBackend } from "../deploy/functions/build"; import { setEnvVarsForEmulators } from "./env"; +import { runWithVirtualEnv } from "../functions/python"; const EVENT_INVOKE = "functions:invoke"; // event name for UA const EVENT_INVOKE_GA4 = "functions_invoke"; // event name GA4 (alphanumertic) @@ -122,15 +124,41 @@ export interface FunctionsEmulatorArgs { projectAlias?: string; } -// FunctionsRuntimeInstance is the handler for a running function invocation +/** + * IPC connection info of a Function Runtime. + */ +export class IPCConn { + constructor(readonly socketPath: string) {} + + httpReqOpts(): http.RequestOptions { + return { + socketPath: this.socketPath, + }; + } +} + +/** + * TCP/IP connection info of a Function Runtime. + */ +export class TCPConn { + constructor(readonly host: string, readonly port: number) {} + + httpReqOpts(): http.RequestOptions { + return { + host: this.host, + port: this.port, + }; + } +} + export interface FunctionsRuntimeInstance { process: ChildProcess; // An emitter which sends our EmulatorLog events from the runtime. events: EventEmitter; // A cwd of the process cwd: string; - // Path to socket file used for HTTP-over-IPC comms. - socketPath: string; + // Communication info for the runtime + conn: IPCConn | TCPConn; } export interface InvokeRuntimeOpts { @@ -353,8 +381,8 @@ export class FunctionsEmulator implements EmulatorInstance { return new Promise((resolve, reject) => { const req = http.request( { + ...worker.runtime.conn.httpReqOpts(), path: `/`, - socketPath: worker.runtime.socketPath, headers: headers, }, resolve @@ -405,6 +433,7 @@ export class FunctionsEmulator implements EmulatorInstance { /.+?[\\\/]node_modules[\\\/].+?/, // Ignore node_modules /(^|[\/\\])\../, // Ignore files which begin the a period /.+\.log/, // Ignore files which have a .log extension + /.+?[\\\/]venv[\\\/].+?/, // Ignore site-packages in venv ], persistent: true, }); @@ -660,26 +689,30 @@ export class FunctionsEmulator implements EmulatorInstance { // In debug mode, we eagerly start the runtime processes to allow debuggers to attach // before invoking a function. if (this.args.debugPort) { - // Since we're about to start a runtime to be shared by all the functions in this codebase, - // we need to make sure it has all the secrets used by any function in the codebase. - emulatableBackend.secretEnv = Object.values( - triggerDefinitions.reduce( - (acc: Record, curr: EmulatedTriggerDefinition) => { - for (const secret of curr.secretEnvironmentVariables || []) { - acc[secret.key] = secret; - } - return acc; - }, - {} - ) - ); - try { - await this.startRuntime(emulatableBackend); - } catch (e: any) { - this.logger.logLabeled( - "ERROR", - `Failed to start functions in ${emulatableBackend.functionsDir}: ${e}` + if (!emulatableBackend.bin?.startsWith("node")) { + this.logger.log("WARN", "--inspect-functions only supported for Node.js runtimes."); + } else { + // Since we're about to start a runtime to be shared by all the functions in this codebase, + // we need to make sure it has all the secrets used by any function in the codebase. + emulatableBackend.secretEnv = Object.values( + triggerDefinitions.reduce( + (acc: Record, curr: EmulatedTriggerDefinition) => { + for (const secret of curr.secretEnvironmentVariables || []) { + acc[secret.key] = secret; + } + return acc; + }, + {} + ) ); + try { + await this.startRuntime(emulatableBackend); + } catch (e: any) { + this.logger.logLabeled( + "ERROR", + `Failed to start functions in ${emulatableBackend.functionsDir}: ${e}` + ); + } } } } @@ -1266,10 +1299,44 @@ export class FunctionsEmulator implements EmulatorInstance { process: childProcess, events: new EventEmitter(), cwd: backend.functionsDir, - socketPath, + conn: new IPCConn(socketPath), }); } + async startPython( + backend: EmulatableBackend, + envs: Record + ): Promise { + const args = ["functions-framework"]; + + if (this.args.debugPort) { + this.logger.log("WARN", "--inspect-functions not supported for Python functions. Ignored."); + } + + // No support generic socket interface for Unix Domain Socket/Named Pipe in the python. + // Use TCP/IP stack instead. + const port = await portfinder.getPortPromise({ + port: 8081 + randomInt(0, 1000), // Add a small jitter to avoid race condition. + }); + const childProcess = runWithVirtualEnv(args, backend.functionsDir, { + ...process.env, + ...envs, + // Required to flush stdout/stderr immediately to the piped channels. + PYTHONUNBUFFERED: "1", + // Required to prevent flask development server to reload on code changes. + DEBUG: "False", + HOST: "127.0.0.1", + PORT: port.toString(), + }); + + return { + process: childProcess, + events: new EventEmitter(), + cwd: backend.functionsDir, + conn: new TCPConn("127.0.0.1", port), + }; + } + async startRuntime( backend: EmulatableBackend, trigger?: EmulatedTriggerDefinition @@ -1277,7 +1344,12 @@ export class FunctionsEmulator implements EmulatorInstance { const runtimeEnv = this.getRuntimeEnvs(backend, trigger); const secretEnvs = await this.resolveSecretEnvs(backend, trigger); - const runtime = await this.startNode(backend, { ...runtimeEnv, ...secretEnvs }); + let runtime; + if (backend.runtime!.startsWith("python")) { + runtime = await this.startPython(backend, { ...runtimeEnv, ...secretEnvs }); + } else { + runtime = await this.startNode(backend, { ...runtimeEnv, ...secretEnvs }); + } const extensionLogInfo = { instanceId: backend.extensionInstanceId, ref: backend.extensionVersion?.ref, diff --git a/src/emulator/functionsRuntimeWorker.ts b/src/emulator/functionsRuntimeWorker.ts index 3358ac68311..5365b7989d2 100644 --- a/src/emulator/functionsRuntimeWorker.ts +++ b/src/emulator/functionsRuntimeWorker.ts @@ -8,6 +8,7 @@ import { EventEmitter } from "events"; import { EmulatorLogger, ExtensionLogInfo } from "./emulatorLogger"; import { FirebaseError } from "../error"; import { Serializable } from "child_process"; +import { IncomingMessage } from "http"; type LogListener = (el: EmulatorLog) => any; @@ -118,12 +119,12 @@ export class RuntimeWorker { return new Promise((resolve) => { const proxy = http.request( { + ...this.runtime.conn.httpReqOpts(), method: req.method, path: req.path, headers: req.headers, - socketPath: this.runtime.socketPath, }, - (_resp) => { + (_resp: IncomingMessage) => { resp.writeHead(_resp.statusCode || 200, _resp.headers); const piped = _resp.pipe(resp); piped.on("finish", () => { @@ -178,20 +179,19 @@ export class RuntimeWorker { isSocketReady(): Promise { return new Promise((resolve, reject) => { - const req = http - .request( - { - method: "GET", - path: "/__/health", - socketPath: this.runtime.socketPath, - }, - () => { - // Set the worker state to IDLE for new work - this.readyForWork(); - resolve(); - } - ) - .end(); + const req = http.request( + { + ...this.runtime.conn.httpReqOpts(), + method: "GET", + path: "/__/health", + }, + () => { + // Set the worker state to IDLE for new work + this.readyForWork(); + resolve(); + } + ); + req.end(); req.on("error", (error) => { reject(error); }); diff --git a/src/functions/python.ts b/src/functions/python.ts new file mode 100644 index 00000000000..f37b821e1d8 --- /dev/null +++ b/src/functions/python.ts @@ -0,0 +1,32 @@ +import * as path from "path"; +import * as spawn from "cross-spawn"; +import * as cp from "child_process"; +import { logger } from "../logger"; + +const DEFAULT_VENV_DIR = "venv"; + +/** + * Spawn a process inside the Python virtual environment if found. + */ +export function runWithVirtualEnv( + commandAndArgs: string[], + cwd: string, + envs: Record, + venvDir = DEFAULT_VENV_DIR +): cp.ChildProcess { + const activateScriptPath = + process.platform === "win32" ? ["Scripts", "activate.bat"] : ["bin", "activate"]; + const venvActivate = path.join(cwd, venvDir, ...activateScriptPath); + const command = process.platform === "win32" ? venvActivate : "source"; + const args = [process.platform === "win32" ? "" : venvActivate, "&&", ...commandAndArgs]; + logger.debug(`Running command with virtualenv: command=${command}, args=${JSON.stringify(args)}`); + + return spawn(command, args, { + shell: true, + cwd, + stdio: [/* stdin= */ "pipe", /* stdout= */ "pipe", /* stderr= */ "pipe", "pipe"], + // Linting disabled since internal types expect NODE_ENV which does not apply to Python runtimes. + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any + env: envs as any, + }); +} diff --git a/src/test/deploy/functions/runtimes/discovery/index.spec.ts b/src/test/deploy/functions/runtimes/discovery/index.spec.ts index 266cce9997a..0a6f07b3f62 100644 --- a/src/test/deploy/functions/runtimes/discovery/index.spec.ts +++ b/src/test/deploy/functions/runtimes/discovery/index.spec.ts @@ -96,12 +96,12 @@ describe("detectFromPort", () => { }); it("passes as smoke test", async () => { - nock("http://localhost:8080").get("/__/functions.yaml").times(5).replyWithError({ + nock("http://127.0.0.1:8080").get("/__/functions.yaml").times(5).replyWithError({ message: "Still booting", code: "ECONNREFUSED", }); - nock("http://localhost:8080").get("/__/functions.yaml").reply(200, YAML_TEXT); + nock("http://127.0.0.1:8080").get("/__/functions.yaml").reply(200, YAML_TEXT); const parsed = await discovery.detectFromPort(8080, "project", "nodejs16"); expect(parsed).to.deep.equal(BUILD); diff --git a/src/test/deploy/functions/runtimes/python/index.spec.ts b/src/test/deploy/functions/runtimes/python/index.spec.ts new file mode 100644 index 00000000000..c16dae73830 --- /dev/null +++ b/src/test/deploy/functions/runtimes/python/index.spec.ts @@ -0,0 +1,45 @@ +import { expect } from "chai"; +import * as sinon from "sinon"; + +import * as python from "../../../../../deploy/functions/runtimes/python"; + +const PROJECT_ID = "test-project"; +const SOURCE_DIR = "/some/path/fns"; + +describe("PythonDelegate", () => { + describe("getPythonBinary", () => { + let platformMock: sinon.SinonStub; + + beforeEach(() => { + platformMock = sinon.stub(process, "platform"); + }); + + afterEach(() => { + platformMock.restore(); + }); + + it("returns specific version of the python binary corresponding to the runtime", () => { + platformMock.value("darwin"); + const requestedRuntime = "python310"; + const delegate = new python.Delegate(PROJECT_ID, SOURCE_DIR, requestedRuntime); + + expect(delegate.getPythonBinary()).to.equal("python3.10"); + }); + + it("returns generic python binary given non-recognized python runtime", () => { + platformMock.value("darwin"); + const requestedRuntime = "python312"; + const delegate = new python.Delegate(PROJECT_ID, SOURCE_DIR, requestedRuntime); + + expect(delegate.getPythonBinary()).to.equal("python"); + }); + + it("always returns version-neutral, python.exe on windows", () => { + platformMock.value("win32"); + const requestedRuntime = "python310"; + const delegate = new python.Delegate(PROJECT_ID, SOURCE_DIR, requestedRuntime); + + expect(delegate.getPythonBinary()).to.equal("python.exe"); + }); + }); +}); diff --git a/src/test/emulators/functionsRuntimeWorker.spec.ts b/src/test/emulators/functionsRuntimeWorker.spec.ts index d0cb0ca557d..00bc983930c 100644 --- a/src/test/emulators/functionsRuntimeWorker.spec.ts +++ b/src/test/emulators/functionsRuntimeWorker.spec.ts @@ -1,7 +1,7 @@ import * as httpMocks from "node-mocks-http"; import * as nock from "nock"; import { expect } from "chai"; -import { FunctionsRuntimeInstance } from "../../emulator/functionsEmulator"; +import { FunctionsRuntimeInstance, IPCConn } from "../../emulator/functionsEmulator"; import { EventEmitter } from "events"; import { RuntimeWorker, @@ -21,7 +21,7 @@ class MockRuntimeInstance implements FunctionsRuntimeInstance { events: EventEmitter = new EventEmitter(); exit: Promise; cwd = "/home/users/dir"; - socketPath = "/path/to/socket/foo.sock"; + conn = new IPCConn("/path/to/socket/foo.sock"); constructor() { this.exit = new Promise((resolve) => {