fix(function engine): 重新使用旧引擎，解决新引擎内存泄露问题；支持 ts 函数编译；

maslow · Jul 23, 2021 · ff468f9 · ff468f9
1 parent 0c3557b
commit ff468f9
Show file tree

Hide file tree

Showing 5 changed files with 221 additions and 108 deletions.
diff --git a/src/lib/faas/engine.ts b/src/lib/faas/engine.ts
@@ -1,104 +1,70 @@
-import * as vm from 'vm'
-import { nanosecond2ms } from '../utils/time'
-import { FunctionConsole } from './console'
-import { FunctionResult, IncomingContext, RequireFuncType, RuntimeContext } from './types'
+/**
+ * @deprecated 老版本的云函数引擎，后面逐渐会被弃用
+ */
 
-
-const require_func: RequireFuncType = (module): any => {
-  // const supported = ['crypto', 'path', 'querystring', 'url', 'lodash', 'moment']
-  // if (supported.includes(module)) { return require(module) as any }
-  return require(module) as any
-}
-
-export class FunctionEngine {
-  buildSandbox(incomingCtx: IncomingContext): RuntimeContext {
-    const fconsole = new FunctionConsole()
-
-    const _module = {
-      exports: {}
-    }
-    return {
-      __context__: incomingCtx.context,
-      module: _module,
-      exports: _module.exports,
-      __runtime_promise: null,
-      console: fconsole,
-      less: incomingCtx.less,
-      cloud: incomingCtx.cloud, 
-      require: require_func,
-      Buffer: Buffer
-    }
-  }
-
-  async run(code: string, incomingCtx: IncomingContext): Promise<FunctionResult> {
-    // 调用前计时
-    const _start_time = process.hrtime.bigint()
-
-    const wrapped = `
-      ${code};
-      if(exports.main && exports.main instanceof Function) {
-        __runtime_promise = exports.main(__context__);
-      } else if(main && main instanceof Function) {
-        __runtime_promise = main(__context__)
-      }
-  `
-
-    const sandbox = this.buildSandbox(incomingCtx)
-    const contextifiedObject = vm.createContext(sandbox)
-    const fconsole = sandbox.console
-    try {
-      // @ts-ignore
-      const funcModule = new vm.SourceTextModule(wrapped, { context: contextifiedObject })
-      await funcModule.link(linker)
-
-      await funcModule.evaluate()
-      const data = await sandbox.__runtime_promise
-
-      // 函数执行耗时
-      const _end_time = process.hrtime.bigint()
-      const time_usage = nanosecond2ms(_end_time - _start_time)
-      return {
-        data,
-        logs: fconsole.logs,
-        time_usage
-      }
-    } catch (error) {
-      fconsole.log(error.message)
-      fconsole.log(error.stack)
-
-      // 函数执行耗时
-      const _end_time = process.hrtime.bigint()
-      const time_usage = nanosecond2ms(_end_time - _start_time)
-      return {
-        error: error,
-        logs: fconsole.logs,
-        time_usage
-      }
-    }
-  }
-}
-
-async function linker(specifier, referencingModule) {
-  if (specifier === 'foo') {
-    // @ts-ignore
-    return new vm.SourceTextModule(`
-      // The "secret" variable refers to the global variable we added to
-      // "contextifiedObject" when creating the context.
-      export default secret;
-    `, { context: referencingModule.context })
-  }
-  // if (specifier === 'path') {
-  const mod = require(specifier)
-  // @ts-ignore
-  return new vm.SourceTextModule(
-    Object.keys(mod)
-      .map((x) => `export const ${x} = import.meta.mod.${x};`)
-      .join('\n'),
-    {
-      initializeImportMeta(meta) {
-        meta.mod = mod
-      },
-      context: referencingModule.context
-    }
-  )
-}
+ import * as vm from 'vm'
+ import { nanosecond2ms } from '../utils/time'
+ import { FunctionConsole } from './console'
+ import { FunctionResult, IncomingContext, RequireFuncType, RuntimeContext } from './types'
+
+ const require_func: RequireFuncType = (module): any => {
+   return require(module) as any
+ }
+
+ export class FunctionEngine {
+
+   async run(code: string, incomingCtx: IncomingContext): Promise<FunctionResult> {
+
+     const fconsole = new FunctionConsole()
+     const wrapped = `
+      ${code}; 
+      const __main__ = exports.main || exports.default
+      if(!__main__) { throw new Error('FunctionExecError: main function not found') }
+      if(!(__main__ instanceof Function)) { throw new Error('FunctionExecError: main function must be callable')}
+      __runtime_promise = __main__(__context__ )
+      `
+
+     const _module = {
+       exports: {}
+     }
+     const sandbox: RuntimeContext = {
+       __context__: incomingCtx.context,
+       module: _module,
+       exports: module.exports,
+       __runtime_promise: null,
+       console: fconsole,
+       less: incomingCtx.less,
+       cloud: incomingCtx.cloud,
+       require: require_func,
+       Buffer: Buffer
+     }
+
+     // 调用前计时
+     const _start_time = process.hrtime.bigint()
+     try {
+       const script = new vm.Script(wrapped)
+       script.runInNewContext(sandbox)
+       const data = await sandbox.__runtime_promise
+       // 函数执行耗时
+       const _end_time = process.hrtime.bigint()
+       const time_usage = nanosecond2ms(_end_time - _start_time)
+       return {
+         data,
+         logs: fconsole.logs,
+         time_usage
+       }
+     } catch (error) {
+       fconsole.log(error.message)
+       fconsole.log(error.stack)
+
+       // 函数执行耗时
+       const _end_time = process.hrtime.bigint()
+       const time_usage = nanosecond2ms(_end_time - _start_time)
+       return {
+         error: error,
+         logs: fconsole.logs,
+         time_usage
+       }
+     }
+   }
+ }
diff --git a/src/lib/faas/engine2.ts b/src/lib/faas/engine2.ts
@@ -0,0 +1,110 @@
+/**
+ * 此函数引擎支持 import 语法，使用 Node 新版本（16）中的 vm.SourceTextModule，支持动态引入依赖。
+ * Warning: 使用过程中，发现 vm.SourceTextModule 存在严重内存泄露问题，追溯至 v8 层面，判定暂非应用层可解决的。
+ * 故退回使用经典 vm 引擎，代码暂留，未来可考虑重新使用或移除。
+ */
+
+import * as vm from 'vm'
+import { nanosecond2ms } from '../utils/time'
+import { FunctionConsole } from './console'
+import { FunctionResult, IncomingContext, RequireFuncType, RuntimeContext } from './types'
+
+
+const require_func: RequireFuncType = (module): any => {
+  // const supported = ['crypto', 'path', 'querystring', 'url', 'lodash', 'moment']
+  // if (supported.includes(module)) { return require(module) as any }
+  return require(module) as any
+}
+
+export class FunctionEngine {
+  buildSandbox(incomingCtx: IncomingContext): RuntimeContext {
+    const fconsole = new FunctionConsole()
+
+    const _module = {
+      exports: {}
+    }
+    return {
+      __context__: incomingCtx.context,
+      module: _module,
+      exports: _module.exports,
+      __runtime_promise: null,
+      console: fconsole,
+      less: incomingCtx.less,
+      cloud: incomingCtx.cloud, 
+      require: require_func,
+      Buffer: Buffer
+    }
+  }
+
+  async run(code: string, incomingCtx: IncomingContext): Promise<FunctionResult> {
+    // 调用前计时
+    const _start_time = process.hrtime.bigint()
+
+    const wrapped = `
+      ${code};
+      if(exports.main && exports.main instanceof Function) {
+        __runtime_promise = exports.main(__context__);
+      } else if(main && main instanceof Function) {
+        __runtime_promise = main(__context__)
+      }
+  `
+
+    const sandbox = this.buildSandbox(incomingCtx)
+    const contextifiedObject = vm.createContext(sandbox)
+    const fconsole = sandbox.console
+    try {
+      // @ts-ignore
+      const funcModule = new vm.SourceTextModule(wrapped, { context: contextifiedObject })
+      await funcModule.link(linker)
+
+      await funcModule.evaluate()
+      const data = await sandbox.__runtime_promise
+
+      // 函数执行耗时
+      const _end_time = process.hrtime.bigint()
+      const time_usage = nanosecond2ms(_end_time - _start_time)
+      return {
+        data,
+        logs: fconsole.logs,
+        time_usage
+      }
+    } catch (error) {
+      fconsole.log(error.message)
+      fconsole.log(error.stack)
+
+      // 函数执行耗时
+      const _end_time = process.hrtime.bigint()
+      const time_usage = nanosecond2ms(_end_time - _start_time)
+      return {
+        error: error,
+        logs: fconsole.logs,
+        time_usage
+      }
+    }
+  }
+}
+
+async function linker(specifier, referencingModule) {
+  if (specifier === 'foo') {
+    // @ts-ignore
+    return new vm.SourceTextModule(`
+      // The "secret" variable refers to the global variable we added to
+      // "contextifiedObject" when creating the context.
+      export default secret;
+    `, { context: referencingModule.context })
+  }
+  // if (specifier === 'path') {
+  const mod = require(specifier)
+  // @ts-ignore
+  return new vm.SourceTextModule(
+    Object.keys(mod)
+      .map((x) => `export const ${x} = import.meta.mod.${x};`)
+      .join('\n'),
+    {
+      initializeImportMeta(meta) {
+        meta.mod = mod
+      },
+      context: referencingModule.context
+    }
+  )
+}
diff --git a/src/lib/faas/invoke.ts b/src/lib/faas/invoke.ts
@@ -5,6 +5,7 @@ import request from 'axios'
 import Config from "../../config"
 import { CloudFunctionStruct, CloudSdkInterface, FunctionContext } from "./types"
 import { getToken, parseToken } from "../utils/token"
+import * as ts from 'typescript'
 
 /**
  * 调用云函数
@@ -13,7 +14,7 @@ export async function invokeFunction(func: CloudFunctionStruct, param: FunctionC
   // const { query, body, auth, requestId } = param
   const engine = new FunctionEngine()
   const cloud = createCloudSdk()
-  const result = await engine.run(func.code, {
+  const result = await engine.run(func.compiledCode, {
     context: param,
     functionName: func.name,
     less: cloud,
@@ -118,3 +119,15 @@ async function _invokeInFunction(name: string, param: FunctionContext) {
 
   return result
 }
+
+/**
+ * 编译云函数(TS) 到 JS
+ * @param {string} source ts 代码字符串
+ */
+ export function compileTsFunction2js(source: string): string {
+  const jscode = ts.transpile(source, {
+    module: ts.ModuleKind.CommonJS,
+    target: ts.ScriptTarget.ES2017
+  })
+  return jscode
+}
diff --git a/src/lib/faas/types.ts b/src/lib/faas/types.ts
@@ -79,10 +79,16 @@ export interface FunctionResult {
 export interface CloudFunctionStruct {
   _id: string,
   name: string,
+  /**
+   * 云函数源代码，通常是 ts
+   */
   code: string,
+  /**
+   * 云函数编译后的代码，通常是 js
+   */
+  compiledCode: string
   enableHTTP: boolean,
   status: number,
-  time_usage: number,
   created_by: number,
   created_at: number
   updated_at: number

diff --git a/src/router/function/index.ts b/src/router/function/index.ts
@@ -2,7 +2,7 @@ import { Request, Response, Router } from 'express'
 import { db } from '../../lib/db'
 import { checkPermission } from '../../lib/api/permission'
 import { getLogger } from '../../lib/logger'
-import { getCloudFunction, invokeFunction } from '../../lib/faas/invoke'
+import { compileTsFunction2js, getCloudFunction, invokeFunction } from '../../lib/faas/invoke'
 import { FunctionContext } from '../../lib/faas/types'
 import * as multer from 'multer'
 import * as path from 'path'
@@ -22,8 +22,15 @@ const uploader = multer({
   })
 })
 
+/**
+ * 调用云函数，支持文件上传
+ */
 FunctionRouter.post('/invoke/:name', uploader.any(), handleInvokeFunction)
-FunctionRouter.all('/:name', uploader.any(), handleInvokeFunction)         // alias for /invoke/:name
+
+/**
+ * 调用云函数，不支持文件上传
+ */
+FunctionRouter.all('/:name', handleInvokeFunction)         // alias for /invoke/:name
 
 async function handleInvokeFunction(req: Request, res: Response) {
   const requestId = req['requestId']
@@ -53,10 +60,12 @@ async function handleInvokeFunction(req: Request, res: Response) {
     return res.send({ code: 1, error: 'function not found', requestId })
   }
 
+  // 未启用 HTTP 访问则拒绝访问（调试模式除外）
   if (!func.enableHTTP && !debug) {
     return res.status(404).send('Not Found')
   }
 
+  // 函数停用则拒绝访问（调试模式除外）
   if (1 !== func.status && !debug) {
     return res.status(404).send('Not Found')
   }
@@ -71,10 +80,19 @@ async function handleInvokeFunction(req: Request, res: Response) {
     auth: req['auth'],
     requestId,
   }
+
+  // 如果是调试模式或者函数未编译，则编译并更新函数
+  if(debug || !func.compiledCode) {
+    func.compiledCode = compileTsFunction2js(func.code)
+    await db.collection('functions')
+      .doc(func._id)
+      .update({ compiledCode: func.compiledCode, updated_at: Date.now()})
+  }
+
   const result = await invokeFunction(func, ctx)
 
   // 将云函数调用日志存储到数据库
-  {
+  if(debug) {
     await db.collection('function_logs')
       .add({
         requestId: requestId,