From 8761a26d529423543d70cf4335f57613133b8d57 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 26 Mar 2023 15:59:05 +0800 Subject: [PATCH] [change] refactor the app engine load func --- .github/workflows/pr-test.yml | 2 +- .github/workflows/unit-test.yml | 2 +- Makefile | 2 +- engine/READE.md | 3 - engine/init_test.go | 24 -- engine/load.go | 496 +++++++++++++++++++++++--------- engine/load_test.go | 33 +-- engine/process.go | 10 +- engine/process_test.go | 4 +- query/query.go | 4 +- share/pack.go | 18 ++ test/utils.go | 3 +- 12 files changed, 404 insertions(+), 197 deletions(-) delete mode 100644 engine/READE.md delete mode 100644 engine/init_test.go create mode 100644 share/pack.go diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 2846dea25c..3e4259022d 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -10,7 +10,7 @@ on: env: YAO_DEV: ${{ github.WORKSPACE }} YAO_ENV: development - YAO_ROOT: ${{ github.WORKSPACE }}/tests + YAO_ROOT: ${{ github.WORKSPACE }}/../app YAO_HOST: 0.0.0.0 YAO_PORT: 5099 YAO_SESSION: "memory" diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index bc0db6c7ab..bb9ad54f13 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -11,7 +11,7 @@ on: env: YAO_DEV: ${{ github.WORKSPACE }} YAO_ENV: development - YAO_ROOT: ${{ github.WORKSPACE }}/tests + YAO_ROOT: ${{ github.WORKSPACE }}/../app YAO_HOST: 0.0.0.0 YAO_PORT: 5099 YAO_SESSION: "memory" diff --git a/Makefile b/Makefile index f35ca9cedd..33bfc09a5b 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ COMMIT := $(shell git log | head -n 1 | awk '{print substr($$2, 0, 12)}') NOW := $(shell date +"%FT%T%z") # ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) -TESTFOLDER := $(shell $(GO) list ./... | grep -E 'api|model|flow|script|fs|i18n|connector|query|plugin|cert|crypto|task|schedule|runtime|helper|utils|widget|importer|store|widgets' | grep -vE 'examples|tests*|config') +TESTFOLDER := $(shell $(GO) list ./... | grep -E 'api|model|flow|script|fs|i18n|connector|query|plugin|cert|crypto|task|schedule|runtime|helper|utils|widget|importer|store|widgets|engine' | grep -vE 'examples|tests*|config') TESTTAGS ?= "" # TESTWIDGETS := $(shell $(GO) list ./widgets/...) diff --git a/engine/READE.md b/engine/READE.md deleted file mode 100644 index 11985dfcff..0000000000 --- a/engine/READE.md +++ /dev/null @@ -1,3 +0,0 @@ -# 程序入口 - -加载各种文件 diff --git a/engine/init_test.go b/engine/init_test.go deleted file mode 100644 index 0f0e76f814..0000000000 --- a/engine/init_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package engine - -import ( - "os" - "testing" - - "github.com/yaoapp/yao/config" -) - -var cfg config.Config - -func TestMain(m *testing.M) { - - // 加载模型等 - Load(config.Conf) - - // Run test suites - exitVal := m.Run() - - // we can do clean up code here - // gou.KillPlugins() - - os.Exit(exitVal) -} diff --git a/engine/load.go b/engine/load.go index de11c7f07f..e1972c73e2 100644 --- a/engine/load.go +++ b/engine/load.go @@ -3,10 +3,10 @@ package engine import ( "fmt" "os" - "path/filepath" "strings" "github.com/fatih/color" + "github.com/yaoapp/gou/application" "github.com/yaoapp/kun/exception" "github.com/yaoapp/yao/api" "github.com/yaoapp/yao/cert" @@ -19,34 +19,37 @@ import ( "github.com/yaoapp/yao/model" "github.com/yaoapp/yao/plugin" "github.com/yaoapp/yao/query" + "github.com/yaoapp/yao/runtime" "github.com/yaoapp/yao/schedule" "github.com/yaoapp/yao/script" "github.com/yaoapp/yao/share" "github.com/yaoapp/yao/socket" "github.com/yaoapp/yao/store" - "github.com/yaoapp/yao/studio" "github.com/yaoapp/yao/task" "github.com/yaoapp/yao/websocket" "github.com/yaoapp/yao/widget" + "github.com/yaoapp/yao/widgets" ) -// Load 根据配置加载 API, FLow, Model, Plugin +// Load application engine func Load(cfg config.Config) (err error) { defer func() { err = exception.Catch(recover()) }() - // // Load Runtime - // err = runtime.Load(cfg) - // if err != nil { - // printErr(cfg.Mode, "Runtime", err) - // } - - // // 加载应用信息 - // // 第一步: 加载应用信息 - // app.Load(cfg) + // SET XGEN_BASE + adminRoot := "yao" + if share.App.Optional != nil { + if root, has := share.App.Optional["adminRoot"]; has { + adminRoot = fmt.Sprintf("%v", root) + } + } + os.Setenv("XGEN_BASE", adminRoot) - // 加密密钥函数 - // model.LoadCrypt(fmt.Sprintf(`{"key":"%s"}`, cfg.DB.AESKey), "AES") - // gou.LoadCrypt(`{}`, "PASSWORD") + // load the application + err = loadApp(cfg.Root) + if err != nil { + printErr(cfg.Mode, "Load Application", err) + panic(err) + } // Load Certs err = cert.Load(cfg) @@ -54,7 +57,7 @@ func Load(cfg config.Config) (err error) { printErr(cfg.Mode, "Cert", err) } - // Load connectors + // Load Connectors err = connector.Load(cfg) if err != nil { printErr(cfg.Mode, "Connector", err) @@ -72,134 +75,100 @@ func Load(cfg config.Config) (err error) { printErr(cfg.Mode, "i18n", err) } - // Load Studio development mode only - if cfg.Mode == "development" { - err = studio.Load(cfg) - if err != nil { - printErr(cfg.Mode, "Studio", err) - } + // start v8 runtime + err = runtime.Start(cfg) + if err != nil { + printErr(cfg.Mode, "Runtime", err) } - // 第二步: 建立数据库 & 会话连接 - err = share.DBConnect(cfg.DB) // 创建数据库连接 + // Make Database connections + err = share.DBConnect(cfg.DB) if err != nil { printErr(cfg.Mode, "DB", err) } - // share.SessionConnect(cfg.Session) // 创建会话服务器链接 - - // 加载应用引擎 - if os.Getenv("YAO_DEV") != "" { - LoadEngine(filepath.Join(os.Getenv("YAO_DEV"), "/yao")) - } else { - LoadEngine() - } - - // 第三步: 加载数据分析引擎 - query.Load(cfg) // 加载数据分析引擎 - - // 第四步: 加载共享库 & JS 处理器 - err = share.Load(cfg) // 加载共享库 lib + // Load Query Engine + err = query.Load(cfg) if err != nil { - printErr(cfg.Mode, "Lib", err) + printErr(cfg.Mode, "Query Engine", err) } - err = script.Load(cfg) // 加载JS处理器 script + // Load Scripts + err = script.Load(cfg) if err != nil { printErr(cfg.Mode, "Script", err) } - // 第五步: 加载数据模型等 - err = model.Load(cfg) // 加载数据模型 model + // Load Models + err = model.Load(cfg) if err != nil { printErr(cfg.Mode, "Model", err) } - err = flow.Load(cfg) // 加载业务逻辑 Flow + // Load Data flows + err = flow.Load(cfg) if err != nil { printErr(cfg.Mode, "Flow", err) } - err = store.Load(cfg) // Load stores + // Load Stores + err = store.Load(cfg) if err != nil { printErr(cfg.Mode, "Store", err) } - err = plugin.Load(cfg) // 加载业务插件 plugin + // Load Plugins + err = plugin.Load(cfg) if err != nil { printErr(cfg.Mode, "Plugin", err) } - // // XGEN 1.0 - // if share.App.XGen == "1.0" { - - // // SET XGEN_BASE - // // adminRoot := "yao" - // // if share.App.Optional != nil { - // // if root, has := share.App.Optional["adminRoot"]; has { - // // adminRoot = fmt.Sprintf("%v", root) - // // } - // // } - // // os.Setenv("XGEN_BASE", adminRoot) - - // // Load build-in widgets - // err = widgets.Load(cfg) - // if err != nil { - // printErr(cfg.Mode, "Widgets", err) - // } - - // delete(gou.APIs, "xiang.table") - // delete(gou.APIs, "xiang.page") - // delete(gou.APIs, "xiang.chart") - // delete(gou.APIs, "xiang.xiang") - // delete(gou.APIs, "xiang.user") - // delete(gou.APIs, "xiang.storage") - - // } else { // old version - // err = table.Load(cfg) // 加载数据表格 table - // if err != nil { - // printErr(cfg.Mode, "Table", err) - // } - - // err = chart.Load(cfg) // 加载分析图表 chart - // if err != nil { - // printErr(cfg.Mode, "Chart", err) - // } - - // err = page.Load(cfg) // 加载页面 page 忽略错误 - // if err != nil { - // printErr(cfg.Mode, "Page", err) - // } - // } - - importer.Load(cfg) // 加载数据导入 imports + // Load WASM Application (experimental) + // Load build-in widgets (table / form / chart / ...) + err = widgets.Load(cfg) + if err != nil { + printErr(cfg.Mode, "Widgets", err) + } + + // Load Importers + err = importer.Load(cfg) + if err != nil { + printErr(cfg.Mode, "Plugin", err) + } + + // Load Apis err = api.Load(cfg) // 加载业务接口 API if err != nil { printErr(cfg.Mode, "API", err) } + // Load Sockets err = socket.Load(cfg) // Load sockets if err != nil { printErr(cfg.Mode, "Socket", err) } - err = websocket.Load(cfg) // Load websockets (client) + // Load websockets (client mode) + err = websocket.Load(cfg) if err != nil { printErr(cfg.Mode, "WebSocket", err) } - err = task.Load(cfg) // Load tasks + // Load tasks + err = task.Load(cfg) if err != nil { printErr(cfg.Mode, "Task", err) } - err = schedule.Load(cfg) // Load schedules + // Load schedules + err = schedule.Load(cfg) if err != nil { printErr(cfg.Mode, "Schedule", err) } - err = widget.Load(cfg) // Load widgets + // Load Custom Widget + err = widget.Load(cfg) if err != nil { printErr(cfg.Mode, "Widget", err) } @@ -207,6 +176,78 @@ func Load(cfg config.Config) (err error) { return nil } +// Unload application engine +func Unload() (err error) { + defer func() { err = exception.Catch(recover()) }() + + // Close DB + err = share.DBClose() + + // Stop Runtime + err = runtime.Stop() + + // Recycle + // api + // models + // flows + // stores + // scripts + // connectors + // filesystem + // i18n + // certs + // plugins + // importers + // tasks + // schedules + // sockets + // websockets + // widgets + // custom widget + + return err +} + +// Reload the application engine +func Reload(cfg config.Config) error { + err := Unload() + if err != nil { + return err + } + return Load(cfg) +} + +// loadApp load the application from bindata / pkg / disk +func loadApp(root string) error { + + var err error + var app application.Application + + if root == "bin:application.pkg" { + app, err = application.OpenFromBin(root, &share.Pack{}) // Load app from Bin + if err != nil { + return err + } + application.Load(app) + + } else if strings.HasSuffix(root, ".pkg") { + + app, err = application.OpenFromPkg(root, &share.Pack{}) // Load app from .pkg file + if err != nil { + return err + } + application.Load(app) + } + + app, err = application.OpenFromDisk(root) // Load app from Disk + if err != nil { + return err + } + + application.Load(app) + return nil +} + func printErr(mode, widget string, err error) { message := fmt.Sprintf("[%s] %s", widget, err.Error()) if !strings.Contains(message, "does not exists") && mode == "development" { @@ -214,53 +255,230 @@ func printErr(mode, widget string, err error) { } } +// // LoadDeprecated 根据配置加载 API, FLow, Model, Plugin +// func LoadDeprecated(cfg config.Config) (err error) { +// defer func() { err = exception.Catch(recover()) }() + +// // // Load Runtime +// // err = runtime.Load(cfg) +// // if err != nil { +// // printErr(cfg.Mode, "Runtime", err) +// // } + +// // // 加载应用信息 +// // // 第一步: 加载应用信息 +// // app.Load(cfg) + +// // 加密密钥函数 +// // model.LoadCrypt(fmt.Sprintf(`{"key":"%s"}`, cfg.DB.AESKey), "AES") +// // gou.LoadCrypt(`{}`, "PASSWORD") + +// // Load Certs +// err = cert.Load(cfg) +// if err != nil { +// printErr(cfg.Mode, "Cert", err) +// } + +// // Load connectors +// err = connector.Load(cfg) +// if err != nil { +// printErr(cfg.Mode, "Connector", err) +// } + +// // Load FileSystem +// err = fs.Load(cfg) +// if err != nil { +// printErr(cfg.Mode, "FileSystem", err) +// } + +// // Load i18n +// err = i18n.Load(cfg) +// if err != nil { +// printErr(cfg.Mode, "i18n", err) +// } + +// // Load Studio development mode only +// if cfg.Mode == "development" { +// err = studio.Load(cfg) +// if err != nil { +// printErr(cfg.Mode, "Studio", err) +// } +// } + +// // 第二步: 建立数据库 & 会话连接 +// err = share.DBConnect(cfg.DB) // 创建数据库连接 +// if err != nil { +// printErr(cfg.Mode, "DB", err) +// } + +// // share.SessionConnect(cfg.Session) // 创建会话服务器链接 + +// // 加载应用引擎 +// if os.Getenv("YAO_DEV") != "" { +// LoadEngine(filepath.Join(os.Getenv("YAO_DEV"), "/yao")) +// } else { +// LoadEngine() +// } + +// // 第三步: 加载数据分析引擎 +// query.Load(cfg) // 加载数据分析引擎 + +// // 第四步: 加载共享库 & JS 处理器 +// err = share.Load(cfg) // 加载共享库 lib +// if err != nil { +// printErr(cfg.Mode, "Lib", err) +// } + +// err = script.Load(cfg) // 加载JS处理器 script +// if err != nil { +// printErr(cfg.Mode, "Script", err) +// } + +// // 第五步: 加载数据模型等 +// err = model.Load(cfg) // 加载数据模型 model +// if err != nil { +// printErr(cfg.Mode, "Model", err) +// } + +// err = flow.Load(cfg) // 加载业务逻辑 Flow +// if err != nil { +// printErr(cfg.Mode, "Flow", err) +// } + +// err = store.Load(cfg) // Load stores +// if err != nil { +// printErr(cfg.Mode, "Store", err) +// } + +// err = plugin.Load(cfg) // 加载业务插件 plugin +// if err != nil { +// printErr(cfg.Mode, "Plugin", err) +// } + +// // // XGEN 1.0 +// // if share.App.XGen == "1.0" { + +// // // SET XGEN_BASE +// // // adminRoot := "yao" +// // // if share.App.Optional != nil { +// // // if root, has := share.App.Optional["adminRoot"]; has { +// // // adminRoot = fmt.Sprintf("%v", root) +// // // } +// // // } +// // // os.Setenv("XGEN_BASE", adminRoot) + +// // // Load build-in widgets +// // err = widgets.Load(cfg) +// // if err != nil { +// // printErr(cfg.Mode, "Widgets", err) +// // } + +// // delete(gou.APIs, "xiang.table") +// // delete(gou.APIs, "xiang.page") +// // delete(gou.APIs, "xiang.chart") +// // delete(gou.APIs, "xiang.xiang") +// // delete(gou.APIs, "xiang.user") +// // delete(gou.APIs, "xiang.storage") + +// // } else { // old version +// // err = table.Load(cfg) // 加载数据表格 table +// // if err != nil { +// // printErr(cfg.Mode, "Table", err) +// // } + +// // err = chart.Load(cfg) // 加载分析图表 chart +// // if err != nil { +// // printErr(cfg.Mode, "Chart", err) +// // } + +// // err = page.Load(cfg) // 加载页面 page 忽略错误 +// // if err != nil { +// // printErr(cfg.Mode, "Page", err) +// // } +// // } + +// importer.Load(cfg) // 加载数据导入 imports + +// err = api.Load(cfg) // 加载业务接口 API +// if err != nil { +// printErr(cfg.Mode, "API", err) +// } + +// err = socket.Load(cfg) // Load sockets +// if err != nil { +// printErr(cfg.Mode, "Socket", err) +// } + +// err = websocket.Load(cfg) // Load websockets (client) +// if err != nil { +// printErr(cfg.Mode, "WebSocket", err) +// } + +// err = task.Load(cfg) // Load tasks +// if err != nil { +// printErr(cfg.Mode, "Task", err) +// } + +// err = schedule.Load(cfg) // Load schedules +// if err != nil { +// printErr(cfg.Mode, "Schedule", err) +// } + +// err = widget.Load(cfg) // Load widgets +// if err != nil { +// printErr(cfg.Mode, "Widget", err) +// } + +// return nil +// } + // Reload 根据配置重新加载 API, FLow, Model, Plugin -func Reload(cfg config.Config) { - // gou.APIs = map[string]*gou.API{} - // gou.Models = map[string]*gou.Model{} - // gou.Flows = map[string]*gou.Flow{} - // gou.Plugins = map[string]*gou.Plugin{} - Load(cfg) -} +// func Reload(cfg config.Config) { +// gou.APIs = map[string]*gou.API{} +// gou.Models = map[string]*gou.Model{} +// gou.Flows = map[string]*gou.Flow{} +// gou.Plugins = map[string]*gou.Plugin{} +// Load(cfg) +// } // LoadEngine 加载引擎的 API, Flow, Model 配置 -func LoadEngine(from ...string) { - // var scripts []share.Script - // if len(from) > 0 { - // scripts = share.GetFilesFS(from[0], ".json") - // } else { - // scripts = share.GetFilesBin("yao", ".json") - // } - - // if scripts == nil { - // exception.New("读取文件失败", 500, from).Throw() - // } - - // if len(scripts) == 0 { - // exception.New("读取文件失败, 未找到任何可执行脚本", 500, from).Throw() - // } - - // // 加载 API, Flow, Models, Table, Chart, Screens - // for _, script := range scripts { - // switch script.Type { - // case "models": - // gou.LoadModel(string(script.Content), "xiang."+script.Name) - // break - // case "flows": - // gou.LoadFlow(string(script.Content), "xiang."+script.Name) - // break - // case "apis": - // gou.LoadAPI(string(script.Content), "xiang."+script.Name) - // break - // } - // } - - // // 加载数据应用 - // for _, script := range scripts { - // switch script.Type { - // case "tables": - // table.LoadTable(string(script.Content), "xiang."+script.Name) - // break - // } - // } -} +// func LoadEngine(from ...string) { +// var scripts []share.Script +// if len(from) > 0 { +// scripts = share.GetFilesFS(from[0], ".json") +// } else { +// scripts = share.GetFilesBin("yao", ".json") +// } + +// if scripts == nil { +// exception.New("读取文件失败", 500, from).Throw() +// } + +// if len(scripts) == 0 { +// exception.New("读取文件失败, 未找到任何可执行脚本", 500, from).Throw() +// } + +// // 加载 API, Flow, Models, Table, Chart, Screens +// for _, script := range scripts { +// switch script.Type { +// case "models": +// gou.LoadModel(string(script.Content), "xiang."+script.Name) +// break +// case "flows": +// gou.LoadFlow(string(script.Content), "xiang."+script.Name) +// break +// case "apis": +// gou.LoadAPI(string(script.Content), "xiang."+script.Name) +// break +// } +// } + +// // 加载数据应用 +// for _, script := range scripts { +// switch script.Type { +// case "tables": +// table.LoadTable(string(script.Content), "xiang."+script.Name) +// break +// } +// } +// } diff --git a/engine/load_test.go b/engine/load_test.go index f47a7efe25..7f9b770500 100644 --- a/engine/load_test.go +++ b/engine/load_test.go @@ -1,35 +1,26 @@ package engine import ( - "os" - "path" "testing" "github.com/stretchr/testify/assert" + "github.com/yaoapp/gou/api" "github.com/yaoapp/yao/config" ) func TestLoad(t *testing.T) { - defer Load(config.Conf) - assert.NotPanics(t, func() { - Load(config.Conf) - }) + defer Unload() + err := Load(config.Conf) + assert.Nil(t, err) + assert.Greater(t, len(api.APIs), 0) } -// 从文件系统载入引擎文件 -func TestLoadEngineFS(t *testing.T) { - defer Load(config.Conf) - root := path.Join(os.Getenv("YAO_DEV"), "/yao") - assert.NotPanics(t, func() { - LoadEngine(root) - }) +func TestReload(t *testing.T) { + defer Unload() + err := Load(config.Conf) + assert.Nil(t, err) -} - -// 从BinDataz载入引擎文件 -func TestLoadEngineBin(t *testing.T) { - defer Load(config.Conf) - assert.NotPanics(t, func() { - LoadEngine() - }) + Reload(config.Conf) + assert.Nil(t, err) + assert.Greater(t, len(api.APIs), 0) } diff --git a/engine/process.go b/engine/process.go index 98a9f2516e..ebf536a20f 100644 --- a/engine/process.go +++ b/engine/process.go @@ -1,7 +1,10 @@ package engine import ( + "fmt" + "github.com/yaoapp/gou/process" + "github.com/yaoapp/yao/config" "github.com/yaoapp/yao/share" ) @@ -30,8 +33,11 @@ func processPing(process *process.Process) interface{} { // processInspect 返回系统信息 func processInspect(process *process.Process) interface{} { - share.App.Icons.Set("favicon", "/api/xiang/favicon.ico") - return share.App.Public() + return map[string]interface{}{ + "VERSION": fmt.Sprintf("%s %s", share.VERSION, share.PRVERSION), + "BUILDNAME": share.BUILDNAME, + "CONFIG": config.Conf, + } } // processFavicon 运行模型 MustCreate diff --git a/engine/process_test.go b/engine/process_test.go index 3fe1b3b240..49ebbab81f 100644 --- a/engine/process_test.go +++ b/engine/process_test.go @@ -22,7 +22,7 @@ func TestProcessAliasPing(t *testing.T) { } func TestProcessInspect(t *testing.T) { - res, ok := process.New("xiang.sys.Inspect").Run().(share.AppInfo) + res, ok := process.New("xiang.sys.Inspect").Run().(map[string]interface{}) assert.True(t, ok) - assert.NotNil(t, res.Version) + assert.NotNil(t, res["VERSION"]) } diff --git a/query/query.go b/query/query.go index bfe6343168..485b9c42aa 100644 --- a/query/query.go +++ b/query/query.go @@ -11,7 +11,7 @@ import ( ) // Load 加载查询引擎 -func Load(cfg config.Config) { +func Load(cfg config.Config) error { if _, has := query.Engines["default"]; !has { registerDefault() @@ -36,6 +36,8 @@ func Load(cfg config.Config) { }) } } + + return nil } // registerDefaultQuery register the default engine diff --git a/share/pack.go b/share/pack.go new file mode 100644 index 0000000000..5ee5b6f0a2 --- /dev/null +++ b/share/pack.go @@ -0,0 +1,18 @@ +package share + +// ******************************************************************************** +// WARNING: DO NOT MODIFY THIS FILE. IT WILL BE REPLACED BY THE APPLICATION CODE. +// ********************************************************************************* + +// Pack the yao app package +type Pack struct{} + +// Decode the package decode method +func (pkg *Pack) Decode(data []byte) ([]byte, error) { + return data, nil +} + +// Encode the package encode method +func (pkg *Pack) Encode(data []byte) ([]byte, error) { + return data, nil +} diff --git a/test/utils.go b/test/utils.go index bb1a6469a1..3c3430cfdd 100644 --- a/test/utils.go +++ b/test/utils.go @@ -34,8 +34,7 @@ func Prepare(t *testing.T, cfg config.Config) { var err error if root == "bin:application.pkg" { - key := os.Getenv("YAO_TEST_PRIVATE_KEY") - app, err = application.OpenFromBin(root, key) // Load app from Bin + app, err = application.OpenFromBin(root, &share.Pack{}) // Load app from Bin if err != nil { t.Fatal(err) }