refactor: impl bucket domain ingress

labring · maslow · Oct 7, 2023 · Sep 20, 2023 · Sep 20, 2023 · Sep 20, 2023
commit 461c603e4aece1dd358d6f077501efc8966a56f6
diff --git a/server/src/gateway/bucket-domain-task.service.ts b/server/src/gateway/bucket-domain-task.service.ts
@@ -1,29 +1,26 @@
 import { Injectable, Logger } from '@nestjs/common'
 import { RegionService } from 'src/region/region.service'
-import { ApisixService } from './apisix.service'
 import * as assert from 'node:assert'
 import { Cron, CronExpression } from '@nestjs/schedule'
 import { ServerConfig, TASK_LOCK_INIT_TIME } from 'src/constants'
 import { SystemDatabase } from 'src/system-database'
 import { BucketDomain } from './entities/bucket-domain'
 import { DomainPhase, DomainState } from './entities/runtime-domain'
+import { BucketGatewayService } from './ingress/bucket-ingress.service'
 
 @Injectable()
 export class BucketDomainTaskService {
   readonly lockTimeout = 30 // in second
-  readonly concurrency = 1 // concurrency count
   private readonly logger = new Logger(BucketDomainTaskService.name)
 
   constructor(
-    private readonly apisixService: ApisixService,
+    private readonly bucketGateway: BucketGatewayService,
     private readonly regionService: RegionService,
   ) {}
 
   @Cron(CronExpression.EVERY_SECOND)
   async tick() {
-    if (ServerConfig.DISABLED_GATEWAY_TASK) {
-      return
-    }
+    if (ServerConfig.DISABLED_GATEWAY_TASK) return
 
     // Phase `Creating` -> `Created`
     this.handleCreatingPhase().catch((err) => {
@@ -76,16 +73,11 @@ export class BucketDomainTaskService {
     const region = await this.regionService.findByAppId(doc.appid)
     assert(region, 'region not found')
 
-    // create route if not exists
-    const id = `bucket-${doc.bucketName}`
-    const route = await this.apisixService.getRoute(region, id)
-    if (!route) {
-      await await this.apisixService.createBucketRoute(
-        region,
-        doc.bucketName,
-        doc.domain,
-      )
-      this.logger.log('bucket route created:' + doc.domain)
+    // create ingress if not exists
+    const ingress = await this.bucketGateway.getIngress(region, doc)
+    if (!ingress) {
+      await this.bucketGateway.createIngress(region, doc)
+      this.logger.log('bucket ingress created:' + doc.domain)
     }
 
     // update phase to `Created`
@@ -125,11 +117,10 @@ export class BucketDomainTaskService {
     assert(region, 'region not found')
 
     // delete route if exists
-    const id = `bucket-${doc.bucketName}`
-    const route = await this.apisixService.getRoute(region, id)
-    if (route) {
-      await this.apisixService.deleteBucketRoute(region, doc.bucketName)
-      this.logger.log('bucket route deleted: ' + doc.bucketName)
+    const ingress = await this.bucketGateway.getIngress(region, doc)
+    if (ingress) {
+      await this.bucketGateway.deleteIngress(region, doc)
+      this.logger.log('bucket ingress deleted: ' + doc.bucketName)
     }
 
     // update phase to `Deleted`

diff --git a/server/src/gateway/gateway.module.ts b/server/src/gateway/gateway.module.ts
@@ -8,6 +8,7 @@ import { BucketDomainTaskService } from './bucket-domain-task.service'
 import { RuntimeDomainTaskService } from './runtime-domain-task.service'
 import { CertificateService } from './certificate.service'
 import { RuntimeGatewayService } from './ingress/runtime-ingress.service'
+import { BucketGatewayService } from './ingress/bucket-ingress.service'
 
 @Module({
   imports: [HttpModule],
@@ -20,6 +21,7 @@ import { RuntimeGatewayService } from './ingress/runtime-ingress.service'
     RuntimeDomainTaskService,
     CertificateService,
     RuntimeGatewayService,
+    BucketGatewayService,
   ],
   exports: [RuntimeDomainService, BucketDomainService],
 })

diff --git a/server/src/gateway/ingress/bucket-ingress.service.ts b/server/src/gateway/ingress/bucket-ingress.service.ts
@@ -3,10 +3,12 @@ import { ClusterService } from 'src/region/cluster/cluster.service'
 import { Region } from 'src/region/entities/region'
 import { BucketDomain } from '../entities/bucket-domain'
 import { GetApplicationNamespace } from 'src/utils/getter'
+import { V1Ingress, V1IngressRule } from '@kubernetes/client-node'
+import { LABEL_KEY_APP_ID } from 'src/constants'
 
 @Injectable()
-export class BucketGateway {
-  private readonly logger = new Logger(BucketGateway.name)
+export class BucketGatewayService {
+  private readonly logger = new Logger(BucketGatewayService.name)
   constructor(private readonly clusterService: ClusterService) {}
 
   getIngressName(bucketDomain: BucketDomain) {
@@ -31,15 +33,70 @@ export class BucketGateway {
     const namespace = GetApplicationNamespace(region, appid)
     const name = this.getIngressName(domain)
 
-    const hosts = [domain.domain]
+    // all bucket request should proxy through runtime
+    const backend = { service: { name: `${appid}`, port: { number: 9000 } } }
 
-    // build rules
-    const backend = { service: { name: `${appid}`, port: { number: 8000 } } }
-    const rules = hosts.map((host) => {
-      return {
-        host,
-        http: { paths: [{ path: '/', pathType: 'Prefix', backend }] },
-      }
-    })
+    // build minio endpoint rule
+    const minioUrl = new URL(region.storageConf.externalEndpoint)
+    const minioEndpointHost = minioUrl.host
+    const minioRule: V1IngressRule = {
+      host: minioEndpointHost,
+      http: {
+        paths: [{ path: `/${domain.bucketName}`, pathType: 'Prefix', backend }],
+      },
+    }
+
+    // build bucket host rule
+    // @deprecated  only use minio endpoint host in the future, reserved to compatible with old version
+    const bucketHost = domain.domain
+    const bucketRule: V1IngressRule = {
+      host: bucketHost,
+      http: {
+        paths: [{ path: '/', pathType: 'Prefix', backend }],
+      },
+    }
+
+    // create ingress
+    const ingressClassName = region.gatewayConf.driver
+    const ingressBody: V1Ingress = {
+      metadata: {
+        name,
+        namespace,
+        annotations: {
+          [LABEL_KEY_APP_ID]: appid,
+          'laf.dev/bucket.name': domain.bucketName,
+          'laf.dev/ingress.type': 'bucket',
+          // apisix ingress annotations
+          'k8s.apisix.apache.org/enable-cors': 'true',
+          'k8s.apisix.apache.org/cors-allow-credential': 'false',
+          'k8s.apisix.apache.org/cors-allow-headers': '*',
+          'k8s.apisix.apache.org/cors-allow-methods': '*',
+          'k8s.apisix.apache.org/cors-allow-origin': '*',
+          'k8s.apisix.apache.org/cors-expose-headers': '*',
+          'k8s.apisix.apache.org/svc-namespace': namespace,
+
+          // k8s nginx ingress annotations
+          // websocket is enabled by default in k8s nginx ingress
+          'nginx.ingress.kubernetes.io/enable-cors': 'true',
+          'nginx.ingress.kubernetes.io/cors-allow-credentials': 'true',
+          'nginx.ingress.kubernetes.io/cors-allow-methods': '*',
+          'nginx.ingress.kubernetes.io/cors-allow-headers': '*',
+          'nginx.ingress.kubernetes.io/cors-expose-headers': '*',
+          'nginx.ingress.kubernetes.io/cors-allow-origin': '*',
+        },
+      },
+      spec: { ingressClassName, rules: [minioRule, bucketRule] },
+    }
+
+    const res = await this.clusterService.createIngress(region, ingressBody)
+    return res
+  }
+
+  async deleteIngress(region: Region, domain: BucketDomain) {
+    const namespace = GetApplicationNamespace(region, domain.appid)
+    const name = this.getIngressName(domain)
+
+    const res = await this.clusterService.deleteIngress(region, name, namespace)
+    return res
   }
 }
diff --git a/server/src/gateway/ingress/runtime-ingress.service.ts b/server/src/gateway/ingress/runtime-ingress.service.ts
@@ -11,12 +11,15 @@ export class RuntimeGatewayService {
   private readonly logger = new Logger(RuntimeGatewayService.name)
   constructor(private readonly clusterService: ClusterService) {}
 
-  async getIngress(region: Region, appid: string) {
-    // get ingress
-    const namespace = GetApplicationNamespace(region, appid)
+  getIngressName(domain: RuntimeDomain) {
+    return domain.appid
+  }
 
+  async getIngress(region: Region, domain: RuntimeDomain) {
     // use appid as ingress name of runtime directly
-    const name = `${appid}`
+    const appid = domain.appid
+    const name = this.getIngressName(domain)
+    const namespace = GetApplicationNamespace(region, appid)
 
     const ingress = await this.clusterService.getIngress(
       region,
@@ -89,12 +92,10 @@ export class RuntimeGatewayService {
     return res
   }
 
-  async deleteIngress(region: Region, appid: string) {
-    const ingress = await this.getIngress(region, appid)
-    if (!ingress) return
-
-    const name = ingress.metadata.name
-    const namespace = ingress.metadata.namespace
+  async deleteIngress(region: Region, domain: RuntimeDomain) {
+    const appid = domain.appid
+    const name = this.getIngressName(domain)
+    const namespace = GetApplicationNamespace(region, appid)
 
     // delete ingress
     const res = await this.clusterService.deleteIngress(region, name, namespace)

diff --git a/server/src/gateway/runtime-domain-task.service.ts b/server/src/gateway/runtime-domain-task.service.ts
@@ -90,7 +90,7 @@ export class RuntimeDomainTaskService {
     assert(region, 'region not found')
 
     // create ingress if not exists
-    const ingress = await this.runtimeGateway.getIngress(region, doc.appid)
+    const ingress = await this.runtimeGateway.getIngress(region, doc)
     if (!ingress) {
       const res = await this.runtimeGateway.createIngress(region, doc)
       this.logger.log('runtime default ingress created: ' + doc.appid)
@@ -162,9 +162,9 @@ export class RuntimeDomainTaskService {
 
     // delete ingress if exists
 
-    const ingress = await this.runtimeGateway.getIngress(region, doc.appid)
+    const ingress = await this.runtimeGateway.getIngress(region, doc)
     if (ingress) {
-      const res = await this.runtimeGateway.deleteIngress(region, doc.appid)
+      const res = await this.runtimeGateway.deleteIngress(region, doc)
       this.logger.log('runtime ingress deleted: ' + doc.appid)
       this.logger.debug(JSON.stringify(res))
     }