feat(fs): support standalone domain for fs bucket;

labring · Dec 27, 2021 · 66e9d7a · 66e9d7a
1 parent 816c1e9
commit 66e9d7a
Show file tree

Hide file tree

Showing 12 changed files with 129 additions and 19 deletions.
diff --git a/deploy-scripts/.env b/deploy-scripts/.env
@@ -1,6 +1,10 @@
 # password of the root user of mongodb
 MONGODB_ROOT_PASSWORD=password123
 
+# the laf.js init root account
+INIT_ROOT_ACCOUNT=root
+INIT_ROOT_ACCOUNT_PASSWORD=password123
+
 # db config of system server
 SYS_DB=sys_db
 SYS_DB_USER=sys_user

diff --git a/deploy-scripts/docker-compose.yml b/deploy-scripts/docker-compose.yml
@@ -36,6 +36,9 @@ services:
       APP_SERVICE_DEPLOY_URL_SCHEMA: ${APP_SERVICE_DEPLOY_URL_SCHEMA}
       STORAGE_SERVICE_API_ENTRYPOINT: http://storage_service:9010
       STORAGE_SERVICE_SECRET: ${SYS_SERVER_SECRET_SALT}
+      STORAGE_SERVICE_DEPLOY_HOST: "fs.${DEPLOY_DOMAIN:?err}:${PUBLISH_PORT:-8080}"
+      INIT_ROOT_ACCOUNT: ${INIT_ROOT_ACCOUNT}
+      INIT_ROOT_ACCOUNT_PASSWORD: ${INIT_ROOT_ACCOUNT_PASSWORD}
     command: dockerize -wait tcp://mongo:27017 node dist/index.js
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
@@ -57,6 +60,7 @@ services:
       - mongo
     environment:
       DEPLOY_DOMAIN: "*.${DEPLOY_DOMAIN:?err}"
+      DEPLOY_FS_DOMAIN: "*.fs.${DEPLOY_DOMAIN:?err}"
       SYS_CLIENT_HOST: ${SYS_CLIENT_HOST:?err}
       DOCS_HOST: "docs.${DEPLOY_DOMAIN:?err}"
     command: "sh /scripts/start.sh"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -37,6 +37,7 @@ services:
       APP_SERVICE_DEPLOY_URL_SCHEMA: 'http'
       STORAGE_SERVICE_API_ENTRYPOINT: http://storage_service:9010
       STORAGE_SERVICE_SECRET: Rewrite_Your_Own_Secret_Salt_abcdefg1234567
+      STORAGE_SERVICE_DEPLOY_HOST: fs.local-dev.host:8080 # `*.local-dev.host` always resolved to 127.0.0.1, used to local development
       DEBUG_BIND_HOST_APP_PATH: '${PWD}/packages/app-service'
       INIT_ROOT_ACCOUNT_PASSWORD: abc123
     command: sh /app/start.sh
@@ -60,7 +61,8 @@ services:
       - mongo
       - system_server
     environment:
-      DEPLOY_DOMAIN: "*.local-dev.host" # `*.local-dev.host` always resolved to 127.0.0.1, used to local development
+      DEPLOY_DOMAIN: "*.local-dev.host"         # `*.local-dev.host` always resolved to 127.0.0.1, used to local development
+      DEPLOY_FS_DOMAIN: "*.fs.local-dev.host"   
       SYS_CLIENT_HOST: console.local-dev.host
       DOCS_HOST: docs.local-dev.host
     volumes:
@@ -70,6 +72,7 @@ services:
       - ./docs/.vitepress/dist:/docs
       - ./packages/gateway/app.conf:/etc/nginx/templates/app.conf.template
       - ./packages/gateway/system.conf:/etc/nginx/templates/system.conf.template
+      - ./packages/gateway/fs-proxy.conf:/etc/nginx/templates/fs-proxy.conf.template
     command: "sh /scripts/start.sh"
     ports:
       - 8080:80

diff --git a/packages/app-console/src/api/file.js b/packages/app-console/src/api/file.js
@@ -2,7 +2,6 @@
 import store from '@/store'
 import { assert } from '@/utils/assert'
 import request from '@/utils/request'
-import { getAppAccessUrl } from '@/api/application'
 
 /**
  * Get bucket list
@@ -164,8 +163,11 @@ export async function deleteFile(bucketName, path, token) {
  * @returns
  */
 export function getAppFileBucketUrl(bucket) {
-  const app_url = getAppAccessUrl()
-  return `${app_url}/file/${bucket}`
+  const appid = store.state.app.appid
+  const domain = store.state.app.storage_deploy_host
+  const schema = store.state.app.storage_deploy_url_schema || 'http'
+  const url = `${schema}://${appid}_${bucket}.${domain}`
+  return url
 }
 
 /**

diff --git a/packages/app-console/src/store/modules/app.js b/packages/app-console/src/store/modules/app.js
@@ -29,7 +29,9 @@ const state = {
   file_token: null,
 
   app_deploy_host: null,
-  app_deploy_url_schema: 'http'
+  app_deploy_url_schema: 'http',
+  storage_deploy_host: null,
+  storage_deploy_url_schema: 'http'
 }
 
 const mutations = {
@@ -55,6 +57,12 @@ const mutations = {
   SET_APP_DEPLOY_URL_SCHEMA: (state, schema) => {
     state.app_deploy_url_schema = schema
   },
+  SET_STORAGE_DEPLOY_HOST: (state, domain) => {
+    state.storage_deploy_host = domain
+  },
+  SET_STORAGE_DEPLOY_URL_SCHEMA: (state, schema) => {
+    state.storage_deploy_url_schema = schema
+  },
   CLEAR_STATE: (state) => {
     state.application = null
     state.appid = null
@@ -64,6 +72,8 @@ const mutations = {
     state.file_token = null
     state.app_deploy_host = null
     state.app_deploy_url_schema = 'http'
+    state.storage_deploy_host = null
+    state.storage_deploy_url_schema = 'http'
   }
 }
 
@@ -81,6 +91,8 @@ const actions = {
     commit('SET_FILE_TOKEN', res.data?.file_token)
     commit('SET_APP_DEPLOY_HOST', res.data?.app_deploy_host)
     commit('SET_APP_DEPLOY_URL_SCHEMA', res.data?.app_deploy_url_schema)
+    commit('SET_STORAGE_DEPLOY_HOST', res.data?.storage_deploy_host)
+    commit('SET_STORAGE_DEPLOY_URL_SCHEMA', res.data?.storage_deploy_url_schema)
   },
   clearStates({ commit }) {
     commit('CLEAR_STATE')

diff --git a/packages/app-console/src/views/storage/files.vue b/packages/app-console/src/views/storage/files.vue
@@ -177,7 +177,7 @@ export default {
         offset: (page - 1) * limit,
         path: this.currentPath,
         token: this.bucketDetail.full_token
-      }).catch(() => { this.listLoading = false })
+      }).finally(() => { this.listLoading = false })
 
       this.list = res.data
 

diff --git a/packages/gateway/Dockerfile b/packages/gateway/Dockerfile
@@ -10,6 +10,7 @@ WORKDIR /app
 EXPOSE 80
 ADD ./system.conf /etc/nginx/templates/system.conf.template
 ADD ./app.conf /etc/nginx/templates/app.conf.template
+ADD ./fs-proxy.conf /etc/nginx/templates/fs-proxy.conf.template
 ADD ./scripts /scripts
 
 ADD ./system-client /app/

diff --git a/packages/gateway/fs-proxy.conf b/packages/gateway/fs-proxy.conf
@@ -1,7 +1,73 @@
 lua_shared_dict domain_cache 128m;
 proxy_cache_path /tmp/cache levels=1:2 keys_zone=file_cache:10m max_size=10g inactive=60m;
 
-# for user custom domain
+# for fs fixed domain
+server {
+   listen 80;
+   server_name ${DEPLOY_FS_DOMAIN};
+
+   gzip on;
+   gzip_min_length 1k;
+   gzip_buffers 4 16k;
+   gzip_http_version 1.0;
+   gzip_comp_level 2;
+   gzip_types text/plain application/x-javascript text/css application/xml;
+
+   location / {
+      # Allow CORS
+      add_header Access-Control-Allow-Origin *;
+      add_header Access-Control-Allow-Methods GET,POST,PUT,DELETE,PATCH,OPTIONS;
+      add_header Access-Control-Allow-Headers *;
+      add_header Access-Control-Expose-Headers *;
+      add_header Access-Control-Max-Age 3600;
+
+      if ($request_method = 'OPTIONS') {
+         return 204;
+      }
+
+      # resolve fs service
+      resolver 127.0.0.11;
+
+      # resolve proxy uri
+      if ($host ~* "(\w{8}(-\w{4}){3}-\w{12}_[\w|\d]{1,32})\.(.+)$") {
+         set $bucket $1;
+         set $proxy_uri $bucket?path=$uri&$query_string;
+      }
+
+      if ($uri ~* "/dir$") {
+         set $proxy_uri $bucket/dir$is_args$query_string;
+      }
+
+      # proxy cache (TODO)
+      proxy_ignore_headers Set-Cookie Cache-Control;
+      proxy_hide_header Cache-Control;
+      proxy_hide_header Set-Cookie;
+      proxy_cache file_cache;
+      proxy_cache_valid 304 5m;
+      # proxy_cache_valid any 0m;
+      proxy_cache_revalidate on;
+      proxy_cache_min_uses 3;
+      proxy_cache_methods GET;
+      proxy_cache_key key=$host$uri;
+      add_header Nginx-Cache "$upstream_cache_status";
+      add_header X-Accept "$http_accept";
+      add_header cache-key "key=$bucket$uri";
+
+      # proxy
+      proxy_pass http://storage_service:9010/$proxy_uri;
+      proxy_read_timeout 600s;
+      add_header bucket $bucket;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+      # Logging
+      log_by_lua_block {
+         ngx.log(ngx.ERR, '<proxy domain>', ngx.var.userdomain, ngx.var.uri);
+      }
+   }
+}
+
+# for user custom domain (TODO)
 server {
    listen 80 default_server;
    server_name _;

diff --git a/packages/system-server/src/api/init.ts b/packages/system-server/src/api/init.ts
@@ -65,7 +65,7 @@ export class InitializerApi {
    * @returns app _id
    */
   static async createSystemExtensionApp(account_id: ObjectId, appid: string) {
-    const SYSTEM_APP_NAME = 'laf.js system server app'
+    const SYSTEM_APP_NAME = 'System Extension Server'
     const db = DatabaseAgent.db
     const db_config = DatabaseAgent.parseConnectionUri(Config.sys_db_uri)
 

diff --git a/packages/system-server/src/config.ts b/packages/system-server/src/config.ts
@@ -143,7 +143,9 @@ export default class Config {
   static get STORAGE_SERVICE_CONFIG() {
     const secret: string = process.env.STORAGE_SERVICE_SECRET
     const entrypoint: string = process.env.STORAGE_SERVICE_API_ENTRYPOINT
-    return { secret, entrypoint }
+    const deploy_host = process.env.STORAGE_SERVICE_DEPLOY_HOST
+    const schema = process.env.STORAGE_SERVICE_DEPLOY_URL_SCHEMA || this.APP_SERVICE_DEPLOY_URL_SCHEMA
+    return { secret, entrypoint, deploy_host, schema }
   }
 
   static get INIT_ROOT_ACCOUNT() {

diff --git a/packages/system-server/src/index.ts b/packages/system-server/src/index.ts
@@ -1,7 +1,7 @@
 /*
  * @Author: Maslow<wangfugen@126.com>
  * @Date: 2021-07-30 10:30:29
- * @LastEditTime: 2021-12-24 13:34:48
+ * @LastEditTime: 2021-12-27 11:18:10
  * @Description: 
  */
 
@@ -11,22 +11,23 @@ import { v4 as uuidv4 } from 'uuid'
 import Config from './config'
 import { router } from './router/index'
 import { logger } from './lib/logger'
+import { DatabaseAgent } from './lib/db-agent'
 
-const server = express()
-server.use(express.json({
+const app = express()
+app.use(express.json({
   limit: '10000kb'
 }) as any)
 
 
-server.all('*', function (_req, res, next) {
+app.all('*', function (_req, res, next) {
   res.header('X-Powered-By', 'LaF Server')
   next()
 })
 
 /**
  * Parsing bearer token
  */
-server.use(function (req, _res, next) {
+app.use(function (req, _res, next) {
   const token = splitBearerToken(req.headers['authorization'] ?? '')
   const auth = parseToken(token) || null
   req['auth'] = auth
@@ -37,9 +38,9 @@ server.use(function (req, _res, next) {
   next()
 })
 
-server.use(router)
+app.use(router)
 
-server.listen(Config.PORT, () => logger.info(`listened on ${Config.PORT}`))
+const server = app.listen(Config.PORT, () => logger.info(`listened on ${Config.PORT}`))
 
 
 process.on('unhandledRejection', (reason, promise) => {
@@ -48,4 +49,15 @@ process.on('unhandledRejection', (reason, promise) => {
 
 process.on('uncaughtException', err => {
   logger.error(`Caught uncaughtException:`, err)
-})
+})
+
+
+process.on('SIGTERM', gracefullyExit)
+process.on('SIGINT', gracefullyExit)
+
+async function gracefullyExit() {
+  DatabaseAgent.sys_accessor.close()
+  server.close(async () => {
+    logger.info('process gracefully exited!')
+  })
+}
diff --git a/packages/system-server/src/router/application/get.ts b/packages/system-server/src/router/application/get.ts
@@ -1,7 +1,7 @@
 /*
  * @Author: Maslow<wangfugen@126.com>
  * @Date: 2021-08-30 15:22:34
- * @LastEditTime: 2021-09-17 16:30:27
+ * @LastEditTime: 2021-12-27 11:39:53
  * @Description: 
  */
 
@@ -76,6 +76,8 @@ export async function handleGetApplicationByAppid(req: Request, res: Response) {
 
   const app_deploy_host = Config.APP_SERVICE_DEPLOY_HOST
   const app_deploy_url_schema = Config.APP_SERVICE_DEPLOY_URL_SCHEMA
+  const storage_deploy_host = Config.STORAGE_SERVICE_CONFIG.deploy_host
+  const storage_deploy_url_schema = Config.STORAGE_SERVICE_CONFIG.schema
 
   app.config = undefined
   return res.send({
@@ -86,7 +88,9 @@ export async function handleGetApplicationByAppid(req: Request, res: Response) {
       debug_token,
       file_token,
       app_deploy_host,
-      app_deploy_url_schema
+      app_deploy_url_schema,
+      storage_deploy_host,
+      storage_deploy_url_schema
     }
   })
 }