feat(k8s): support k8s resources limit for app;

labring · Jan 31, 2022 · ffe0e28 · ffe0e28
1 parent 3be5890
commit ffe0e28
Show file tree

Hide file tree

Showing 9 changed files with 80 additions and 27 deletions.
diff --git a/.gitignore b/.gitignore
@@ -12,4 +12,6 @@ ecosystem.config.js
 coverage
 .nyc_output
 
-kubernetes-dev/mongo-data
+kubernetes-dev/mongo-data
+
+kubernetes-local/
diff --git a/packages/system-client/src/views/application/index.vue b/packages/system-client/src/views/application/index.vue
@@ -460,8 +460,8 @@ export default {
       return version || 'unknown'
     },
     getRuntimeMemory(app) {
-      const memory = app.runtime?.metrics?.memory
-      return memory || '256'
+      const memory = app.runtime?.resources?.limit_memory
+      return memory || '-'
     }
   }
 }

diff --git a/packages/system-server/.gitignore b/packages/system-server/.gitignore
@@ -6,6 +6,7 @@ data/*
 tmp
 
 .env
+.env.local
 
 ecosystem.config.js
 .DS_Store
diff --git a/packages/system-server/src/api/application.ts b/packages/system-server/src/api/application.ts
@@ -1,7 +1,7 @@
 /*
  * @Author: Maslow<wangfugen@126.com>
  * @Date: 2021-08-28 22:00:45
- * @LastEditTime: 2022-01-13 13:48:28
+ * @LastEditTime: 2022-02-01 00:04:33
  * @Description: Application APIs
  */
 
@@ -38,9 +38,20 @@ export interface ApplicationStruct {
   }
   runtime: {
     image: string
-    metrics: {
+    /** @deprecated use `resources` instead, will be removed in future */
+    metrics?: {
       cpu_shares?: number
       memory?: number
+    },
+    resources: {
+      /** `requests.cpu` in kubernetes, in millicores.  ex. `1000` for 1 cpu, `100` for 0.1 cpu.  */
+      req_cpu: string,
+      /** `requests.memory` in kubernetes, in Mib. */
+      req_memory: string,
+      /** `limits.cpu` in kubernetes, also valid for docker cpu shares. In millicores.  ex. `1000` for 1 cpu, `100` for 0.1 cpu. */
+      limit_cpu: string,
+      /** `limits.memory` in kubernetes, also valid for docker memory limit, in Mib. */
+      limit_memory: string
     }
   }
   buckets: {

diff --git a/packages/system-server/src/api/init.ts b/packages/system-server/src/api/init.ts
@@ -83,9 +83,11 @@ export class InitializerApi {
       },
       runtime: {
         image: Config.APP_SERVICE_IMAGE,
-        metrics: {
-          cpu_shares: 2048,
-          memory: 1024
+        resources: { 
+          req_cpu: '100',
+          req_memory: '256',
+          limit_cpu: '1000',
+          limit_memory: '1024'
         }
       },
       buckets: [],

diff --git a/packages/system-server/src/config.ts b/packages/system-server/src/config.ts
@@ -119,16 +119,35 @@ export default class Config {
 
   /**
    * The max memory limit of application service, default is 256 in mb
+   * 
+   * @deprecated this field was deprecated and removed, use APP_SERVICE_DEFAULT_LIMIT_MEMORY instead
+   */
+  static get APP_SERVICE_MEMORY_LIMIT() {
+    throw new Error('APP_SERVICE_MEMORY_LIMIT is deprecated, use APP_SERVICE_DEFAULT_LIMIT_MEMORY instead.')
+  }
+
+  /**
+   * The max cpu limit of application service, default is 100
+   * 
+   * * @deprecated this field was deprecated and removed, use APP_SERVICE_DEFAULT_LIMIT_CPU instead
    */
-  static get APP_SERVICE_MEMORY_LIMIT(): number {
-    return (process.env.APP_SERVICE_MEMORY_LIMIT ?? 256) as number
+  static get APP_SERVICE_CPU_LIMIT(): number {
+    throw new Error('APP_SERVICE_CPU_LIMIT is deprecated, use APP_SERVICE_DEFAULT_LIMIT_CPU instead.')
   }
 
   /**
-   * The cpu shares of application service, default is 256
+   * The resources (cpu & memory) limit of application service. 
    */
-  static get APP_SERVICE_CPU_SHARES(): number {
-    return (process.env.APP_SERVICE_MEMORY_LIMIT ?? 256) as number
+  static get APP_DEFAULT_RESOURCES(): { req_cpu: string, req_memory: string, limit_cpu: string, limit_memory: string } {
+    const limit_cpu = process.env.APP_SERVICE_DEFAULT_LIMIT_CPU ?? '100'
+    const limit_memory = process.env.APP_SERVICE_DEFAULT_LIMIT_MEMORY || '256'
+
+    return { 
+      req_cpu: process.env.APP_SERVICE_DEFAULT_REQUEST_CPU ?? limit_cpu,
+      req_memory: process.env.APP_SERVICE_DEFAULT_REQUEST_MEMORY ?? limit_memory,
+      limit_cpu,
+      limit_memory,
+    }
   }
 
   /**
@@ -173,4 +192,12 @@ export default class Config {
     const package_: string = process.env.SYSTEM_EXTENSION_SERVER_APP_PACKAGE || default_
     return package_
   }
+
+  static get KUBE_NAMESPACE_OF_APP_SERVICES() {
+    return process.env.KUBE_NAMESPACE_OF_APP_SERVICES || 'laf'
+  }
+
+  static get KUBE_NAMESPACE_OF_SYS_SERVICES() {
+    return process.env.KUBE_NAMESPACE_OF_SYS_SERVICES || 'laf'
+  }
 }
diff --git a/packages/system-server/src/lib/service-driver/container.ts b/packages/system-server/src/lib/service-driver/container.ts
@@ -92,10 +92,9 @@ export class DockerContainerServiceDriver implements ServiceDriverInterface {
    */
   private async createService(app: ApplicationStruct) {
     const uri = getApplicationDbUri(app)
-    const memoryLimit = app.runtime?.metrics?.memory ?? Config.APP_SERVICE_MEMORY_LIMIT
+    const memoryLimit = parseInt(app.runtime?.resources?.limit_memory ?? Config.APP_DEFAULT_RESOURCES.limit_memory)
     const max_old_space_size = ~~(memoryLimit * 0.8)
-    // if no cpu-shares set, use memory limit as it
-    const cpuShares = app.runtime?.metrics?.cpu_shares ?? Config.APP_SERVICE_CPU_SHARES
+    const cpuShares = parseInt(app.runtime?.resources?.limit_cpu ?? Config.APP_DEFAULT_RESOURCES.limit_cpu)
     const imageName = app.runtime?.image ?? Config.APP_SERVICE_IMAGE
     const logLevel = Config.LOG_LEVEL
 

diff --git a/packages/system-server/src/lib/service-driver/kubernetes.ts b/packages/system-server/src/lib/service-driver/kubernetes.ts
@@ -7,7 +7,7 @@ import { ServiceDriverInterface } from './interface'
 
 
 export class KubernetesServiceDriver implements ServiceDriverInterface {
-  namespace = "laf"
+  namespace = Config.KUBE_NAMESPACE_OF_APP_SERVICES
   apps_api: k8s.AppsV1Api
   core_api: k8s.CoreV1Api
 
@@ -92,13 +92,18 @@ export class KubernetesServiceDriver implements ServiceDriverInterface {
    */
   private async createK8sDeployment(app: ApplicationStruct, labels: any) {
     const uri = getApplicationDbUri(app)
-    const memoryLimit = app.runtime?.metrics?.memory ?? Config.APP_SERVICE_MEMORY_LIMIT
-    const max_old_space_size = ~~(memoryLimit * 0.8)
-    // if no cpu-shares set, use memory limit as it
-    const cpuShares = app.runtime?.metrics?.cpu_shares ?? Config.APP_SERVICE_CPU_SHARES
+
+    const limit_memory = parseInt(app.runtime?.resources?.limit_memory ?? Config.APP_DEFAULT_RESOURCES.limit_memory)
+    const limit_cpu = app.runtime?.resources?.limit_cpu ?? Config.APP_DEFAULT_RESOURCES.limit_cpu
+
+    const req_cpu = app.runtime?.resources?.req_cpu ?? Config.APP_DEFAULT_RESOURCES.req_cpu
+    const req_memory = app.runtime?.resources?.req_memory ?? Config.APP_DEFAULT_RESOURCES.req_memory
+
     const imageName = app.runtime?.image ?? Config.APP_SERVICE_IMAGE
+    const max_old_space_size = ~~(limit_memory * 0.8)
     const logLevel = Config.LOG_LEVEL
 
+
     // create k8s deployment
     const { body: deployment } = await this.apps_api.createNamespacedDeployment(this.namespace, {
       metadata: {
@@ -133,9 +138,13 @@ export class KubernetesServiceDriver implements ServiceDriverInterface {
                 ],
                 ports: [{ containerPort: 8000 }],
                 resources: {
+                  requests: {
+                    memory: `${req_memory}Mi`,
+                    cpu: `${req_cpu}m`
+                  },
                   limits: {
-                    memory: `${memoryLimit}Mi`,
-                    cpu: `${cpuShares}m`
+                    memory: `${limit_memory}Mi`,
+                    cpu: `${limit_cpu}m`
                   }
                 }
               }

diff --git a/packages/system-server/src/router/application/create.ts b/packages/system-server/src/router/application/create.ts
@@ -1,7 +1,7 @@
 /*
  * @Author: Maslow<wangfugen@126.com>
  * @Date: 2021-08-31 15:00:04
- * @LastEditTime: 2021-12-07 13:57:21
+ * @LastEditTime: 2022-01-31 23:44:45
  * @Description: 
  */
 
@@ -59,9 +59,11 @@ export async function handleCreateApplication(req: Request, res: Response) {
     },
     runtime: {
       image: Config.APP_SERVICE_IMAGE,
-      metrics: {
-        cpu_shares: Config.APP_SERVICE_CPU_SHARES,
-        memory: Config.APP_SERVICE_MEMORY_LIMIT
+      resources: {
+        req_cpu: Config.APP_DEFAULT_RESOURCES.req_cpu,
+        req_memory: Config.APP_DEFAULT_RESOURCES.req_memory,
+        limit_cpu: Config.APP_DEFAULT_RESOURCES.limit_cpu,
+        limit_memory: Config.APP_DEFAULT_RESOURCES.limit_memory,
       }
     },
     buckets: [],