From b921cce2d9eca28531d9a75ecaf3f7e7798fcb6e Mon Sep 17 00:00:00 2001 From: Yuxing Deng Date: Fri, 5 Aug 2022 17:40:21 +0800 Subject: [PATCH] feat(airgap): Basic airgap cli function As design, following actions are supported: - create - update - remove - ls - import - export - update-install-script The downloaded resource will be stored in package dir under CfgPath. The design can be found in https://github.com/cnrancher/autok3s/discussions/480. --- .gitignore | 5 +- cmd/airgap/airgap.go | 26 ++ cmd/airgap/args.go | 44 ++++ cmd/airgap/create.go | 95 ++++++++ cmd/airgap/export.go | 72 ++++++ cmd/airgap/import.go | 82 +++++++ cmd/airgap/ls.go | 56 +++++ cmd/airgap/remove.go | 46 ++++ cmd/airgap/update.go | 110 +++++++++ cmd/airgap/update_script.go | 30 +++ cmd/serve.go | 1 + hack/make-rules/autok3s.sh | 7 + main.go | 5 +- pkg/airgap/archs.go | 42 ++++ pkg/airgap/download.go | 475 ++++++++++++++++++++++++++++++++++++ pkg/airgap/download_test.go | 103 ++++++++ pkg/airgap/tarfiles.go | 164 +++++++++++++ pkg/common/common.go | 2 + pkg/common/db.go | 9 +- pkg/common/metrics.go | 4 +- pkg/common/package.go | 59 +++++ pkg/common/package_test.go | 1 + pkg/settings/script/main.go | 38 +++ pkg/settings/script_prod.go | 21 ++ pkg/settings/setting.go | 35 ++- pkg/types/autok3s.go | 13 +- pkg/utils/util.go | 6 + 27 files changed, 1541 insertions(+), 10 deletions(-) create mode 100644 cmd/airgap/airgap.go create mode 100644 cmd/airgap/args.go create mode 100644 cmd/airgap/create.go create mode 100644 cmd/airgap/export.go create mode 100644 cmd/airgap/import.go create mode 100644 cmd/airgap/ls.go create mode 100644 cmd/airgap/remove.go create mode 100644 cmd/airgap/update.go create mode 100644 cmd/airgap/update_script.go create mode 100644 pkg/airgap/archs.go create mode 100644 pkg/airgap/download.go create mode 100644 pkg/airgap/download_test.go create mode 100644 pkg/airgap/tarfiles.go create mode 100644 pkg/common/package.go create mode 100644 pkg/common/package_test.go create mode 100644 pkg/settings/script/main.go create mode 100644 pkg/settings/script_prod.go diff --git a/.gitignore b/.gitignore index e80c95fa..ef71d69e 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,7 @@ static/ upx*.tar.xz upx*/ .vscode/ -.dapper \ No newline at end of file +.dapper + +# Others +pkg/settings/install.sh diff --git a/cmd/airgap/airgap.go b/cmd/airgap/airgap.go new file mode 100644 index 00000000..d9106c53 --- /dev/null +++ b/cmd/airgap/airgap.go @@ -0,0 +1,26 @@ +package airgap + +import ( + "github.com/spf13/cobra" +) + +var ( + airgap = &cobra.Command{ + Use: "airgap", + Short: "The airgap packages management.", + Long: "The airgap command manages the airgap package for k3s.", + } +) + +func Command() *cobra.Command { + airgap.AddCommand( + listCmd, + createCmd, + removeCmd, + updateCmd, + importCmd, + exportCmd, + updateScriptCmd, + ) + return airgap +} diff --git a/cmd/airgap/args.go b/cmd/airgap/args.go new file mode 100644 index 00000000..98e9d23f --- /dev/null +++ b/cmd/airgap/args.go @@ -0,0 +1,44 @@ +package airgap + +import ( + "fmt" + + pkgairgap "github.com/cnrancher/autok3s/pkg/airgap" + "github.com/cnrancher/autok3s/pkg/common" + + "github.com/AlecAivazis/survey/v2" + "k8s.io/apimachinery/pkg/util/validation" +) + +var ( + airgapFlags = flags{} +) + +type flags struct { + isForce bool + isJSON bool + K3sVersion string + Archs []string +} + +func getArchSelect(def []string) *survey.MultiSelect { + return &survey.MultiSelect{ + Default: def, + Message: "What arch do you prefer?", + Options: pkgairgap.GetValidatedArchs(), + } +} + +func validateName(name string) error { + if name == "" { + return errNameRequire + } + pkgs, _ := common.DefaultDB.ListPackages(&name) + if len(pkgs) > 0 { + return fmt.Errorf("package name %s already exists", name) + } + if errs := validation.IsDNS1123Subdomain(name); len(errs) > 0 { + return fmt.Errorf("name is not validated %s, %v", name, errs) + } + return nil +} diff --git a/cmd/airgap/create.go b/cmd/airgap/create.go new file mode 100644 index 00000000..6a3f8636 --- /dev/null +++ b/cmd/airgap/create.go @@ -0,0 +1,95 @@ +package airgap + +import ( + "sort" + "strings" + + pkgairgap "github.com/cnrancher/autok3s/pkg/airgap" + "github.com/cnrancher/autok3s/pkg/common" + "github.com/cnrancher/autok3s/pkg/utils" + + "github.com/AlecAivazis/survey/v2" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +var createCmd = &cobra.Command{ + Use: "create ", + Short: "Create a new airgap package and will download related resources from internet.", + Args: cobra.ExactArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + sort.Strings(airgapFlags.Archs) + }, + RunE: create, +} + +func init() { + createCmd.Flags().StringVarP(&airgapFlags.K3sVersion, "k3s-version", "v", airgapFlags.K3sVersion, "The version of k3s to store airgap resources.") + createCmd.Flags().StringArrayVar(&airgapFlags.Archs, "arch", airgapFlags.Archs, "The archs of the k3s version. Following archs are support: "+strings.Join(pkgairgap.GetValidatedArchs(), ",")+".") +} + +func create(cmd *cobra.Command, args []string) error { + name := args[0] + if err := validateName(name); err != nil { + return err + } + + var qs []*survey.Question + + if airgapFlags.K3sVersion == "" { + if !utils.IsTerm() { + return errors.New("k3s-version flags is required") + } + qs = append(qs, &survey.Question{ + Name: "k3sVersion", + Prompt: &survey.Input{Message: "K3s Version?"}, + Validate: survey.Required, + }) + } + + if len(airgapFlags.Archs) == 0 { + if !utils.IsTerm() { + return errors.New("at least one arch should be specified") + } + qs = append(qs, &survey.Question{ + Name: "archs", + Prompt: getArchSelect([]string{}), + Validate: survey.Required, + }) + } + + if err := survey.Ask(qs, &airgapFlags); err != nil { + return err + } + + if err := pkgairgap.ValidateArchs(airgapFlags.Archs); err != nil { + return err + } + + pkg := common.Package{ + Name: name, + K3sVersion: airgapFlags.K3sVersion, + Archs: airgapFlags.Archs, + State: common.PackageOutOfSync, + } + + if err := common.DefaultDB.SavePackage(pkg); err != nil { + return err + } + cmd.Printf("airgap package %s record created, prepare to download\n", pkg.Name) + downloader, err := pkgairgap.NewDownloader(pkg) + if err != nil { + return errors.Wrapf(err, "failed to start download process for package %s", pkg.Name) + } + path, err := downloader.DownloadPackage() + if err != nil { + return errors.Wrapf(err, "failed to download package %s", pkg.Name) + } + pkg.FilePath = path + pkg.State = common.PackageActive + if err := common.DefaultDB.SavePackage(pkg); err != nil { + return err + } + cmd.Printf("airgap package %s created and stored in path %s\n", name, path) + return nil +} diff --git a/cmd/airgap/export.go b/cmd/airgap/export.go new file mode 100644 index 00000000..cb5bfb6d --- /dev/null +++ b/cmd/airgap/export.go @@ -0,0 +1,72 @@ +package airgap + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + pkgairgap "github.com/cnrancher/autok3s/pkg/airgap" + "github.com/cnrancher/autok3s/pkg/common" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + "gorm.io/gorm" +) + +var ( + exportCmd = &cobra.Command{ + Use: "export ", + Short: "export package to a tar.gz file, path can be a specific filename or a directory.", + Args: cobra.ExactArgs(2), + RunE: export, + } + errPathInvalid = errors.New("path should be an existing directory or a file with .tar.gz/tgz suffix") +) + +func export(cmd *cobra.Command, args []string) error { + name := args[0] + pkgs, err := common.DefaultDB.ListPackages(&name) + if err == gorm.ErrRecordNotFound { + return fmt.Errorf("package %s not found", name) + } + if err != nil { + return err + } + targetPackage := pkgs[0] + if targetPackage.State != common.PackageActive { + return fmt.Errorf("package %s is not active, airgap resources maybe missing", name) + } + + path := args[1] + + info, err := os.Lstat(path) + if err != nil && !os.IsNotExist(err) { + return err + } + + // check file name if path not exist. + if os.IsNotExist(err) { + base := filepath.Base(path) + if !strings.HasSuffix(base, ".tgz") && + !strings.HasSuffix(base, ".tar.gz") { + return errPathInvalid + } + if _, err := os.Lstat(filepath.Dir(path)); err != nil { + return err + } + // here means path is a file with tar.gz/tgz suffix and parent dir exists + } else if !info.IsDir() { + // here means that input path is a regular file and should return error + return errPathInvalid + } else { + // here means that the input path is a dir and will use package name as the output name + path = filepath.Join(path, name+".tar.gz") + } + + if err := pkgairgap.TarAndGzip(targetPackage.FilePath, path); err != nil { + return err + } + cmd.Printf("package %s export to %s succeed\n", name, path) + return nil +} diff --git a/cmd/airgap/import.go b/cmd/airgap/import.go new file mode 100644 index 00000000..955501c6 --- /dev/null +++ b/cmd/airgap/import.go @@ -0,0 +1,82 @@ +package airgap + +import ( + "errors" + "os" + + pkgairgap "github.com/cnrancher/autok3s/pkg/airgap" + "github.com/cnrancher/autok3s/pkg/common" + + "github.com/AlecAivazis/survey/v2" + "github.com/spf13/cobra" +) + +var ( + importCmd = &cobra.Command{ + Use: "import [name]", + Short: "Import an existing tar.gz file of airgap package. Please refer to export command", + Args: func(cmd *cobra.Command, args []string) error { + if err := cobra.MaximumNArgs(2)(cmd, args); err != nil { + return err + } + if err := cobra.MinimumNArgs(1)(cmd, args); err != nil { + return err + } + return nil + }, + RunE: importFunc, + } + errNameRequire = errors.New("name is required for importing a airgap package") +) + +func importFunc(cmd *cobra.Command, args []string) error { + path := args[0] + var name string + var err error + if len(args) < 2 { + name, err = askForName() + if err != nil { + return err + } + } else { + name = args[1] + } + + if name == "" { + return errNameRequire + } + tmpPath, err := pkgairgap.SaveToTmp(path, name) + if err != nil { + return err + } + defer os.RemoveAll(tmpPath) + + toSave, err := pkgairgap.VerifyFiles(tmpPath) + if err != nil { + return err + } + + toSave.Name = name + toSave.FilePath = pkgairgap.PackagePath(name) + if err := os.Rename(tmpPath, toSave.FilePath); err != nil { + return err + } + toSave.State = common.PackageActive + if err := common.DefaultDB.SavePackage(*toSave); err != nil { + _ = os.RemoveAll(toSave.FilePath) + return err + } + cmd.Printf("package %s imported from %s\n", name, path) + return nil +} + +func askForName() (string, error) { + rtn := "" + err := survey.AskOne(&survey.Input{ + Message: "Please input the package name", + }, &rtn, survey.WithValidator(func(ans interface{}) error { + name, _ := ans.(string) + return validateName(name) + })) + return rtn, err +} diff --git a/cmd/airgap/ls.go b/cmd/airgap/ls.go new file mode 100644 index 00000000..c082a91d --- /dev/null +++ b/cmd/airgap/ls.go @@ -0,0 +1,56 @@ +package airgap + +import ( + "encoding/json" + "os" + "strings" + + "github.com/cnrancher/autok3s/pkg/common" + + "github.com/olekukonko/tablewriter" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +var ( + listCmd = &cobra.Command{ + Use: "ls", + Short: "List all stored airgap packages.", + RunE: list, + } +) + +func init() { + listCmd.Flags().BoolVarP(&airgapFlags.isJSON, "json", "j", airgapFlags.isJSON, "json output") +} + +func list(cmd *cobra.Command, args []string) error { + pkgs, err := common.DefaultDB.ListPackages(nil) + if err != nil { + return errors.Wrap(err, "failed to list airgap packages.") + } + if airgapFlags.isJSON { + data, err := json.Marshal(pkgs) + if err != nil { + return err + } + cmd.Printf("%s\n", string(data)) + return nil + } + table := tablewriter.NewWriter(os.Stdout) + table.SetBorder(false) + table.SetHeaderLine(false) + table.SetColumnSeparator("") + table.SetAlignment(tablewriter.ALIGN_LEFT) + table.SetHeader([]string{"Name", "K3sVersion", "Archs", "State"}) + for _, pkg := range pkgs { + table.Append([]string{ + pkg.Name, + pkg.K3sVersion, + strings.Join(pkg.Archs, ","), + string(pkg.State), + }) + } + table.Render() + return nil +} diff --git a/cmd/airgap/remove.go b/cmd/airgap/remove.go new file mode 100644 index 00000000..3a479b20 --- /dev/null +++ b/cmd/airgap/remove.go @@ -0,0 +1,46 @@ +package airgap + +import ( + "errors" + "fmt" + + pkgairgap "github.com/cnrancher/autok3s/pkg/airgap" + "github.com/cnrancher/autok3s/pkg/common" + "github.com/cnrancher/autok3s/pkg/utils" + + "github.com/spf13/cobra" +) + +var removeCmd = &cobra.Command{ + Use: "remove ", + Short: "Remove a stored airgap package.", + Args: cobra.ExactArgs(1), + RunE: remove, +} + +func init() { + removeCmd.Flags().BoolVarP(&airgapFlags.isForce, "force", "f", false, "Force to delete a package.") +} + +func remove(cmd *cobra.Command, args []string) error { + name := args[0] + if !airgapFlags.isForce { + if !utils.IsTerm() { + return errors.New("please using --force to delete a package") + } + if !utils.AskForConfirmation(fmt.Sprintf("are you going to remove package %s", name), false) { + return nil + } + } + + if err := pkgairgap.RemovePackage(name); err != nil { + return err + } + + if err := common.DefaultDB.DeletePackage(name); err != nil { + return err + } + + cmd.Printf("package %s removed\n", name) + return nil +} diff --git a/cmd/airgap/update.go b/cmd/airgap/update.go new file mode 100644 index 00000000..1687f159 --- /dev/null +++ b/cmd/airgap/update.go @@ -0,0 +1,110 @@ +package airgap + +import ( + "fmt" + "sort" + "strings" + + pkgairgap "github.com/cnrancher/autok3s/pkg/airgap" + "github.com/cnrancher/autok3s/pkg/common" + "github.com/cnrancher/autok3s/pkg/utils" + + "github.com/AlecAivazis/survey/v2" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "gorm.io/gorm" +) + +var updateCmd = &cobra.Command{ + Use: "update ", + Short: "Update a stored package with new selected archs.", + Args: cobra.ExactArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + sort.Strings(airgapFlags.Archs) + }, + RunE: update, +} + +func init() { + updateCmd.Flags().StringVarP(&airgapFlags.K3sVersion, "k3s-version", "v", airgapFlags.K3sVersion, "The version of k3s to store airgap resources.") + updateCmd.Flags().StringArrayVar(&airgapFlags.Archs, "arch", airgapFlags.Archs, "The archs of the k3s version. Following archs are support: "+strings.Join(pkgairgap.GetValidatedArchs(), ",")+".") + updateCmd.Flags().BoolVarP(&airgapFlags.isForce, "force", "f", false, "Force update without comfirm and skip state check") +} + +func update(cmd *cobra.Command, args []string) error { + name := args[0] + pkgs, err := common.DefaultDB.ListPackages(&name) + if err == gorm.ErrRecordNotFound { + return fmt.Errorf("package %s not found", name) + } + if err != nil { + return err + } + toUpdate := pkgs[0] + + versionChanged := airgapFlags.K3sVersion != "" && airgapFlags.K3sVersion != toUpdate.K3sVersion + + if len(airgapFlags.Archs) == 0 { + if !utils.IsTerm() { + return errors.New("at least one arch is required for updating airgap package") + } + if err := survey.AskOne(getArchSelect(toUpdate.Archs), &airgapFlags.Archs); err != nil { + return err + } + } + add, del := pkgairgap.GetArchDiff(toUpdate.Archs, airgapFlags.Archs) + + if !airgapFlags.isForce && !utils.IsTerm() { + if versionChanged { + return errors.New("k3s version is changed, you must use -f flag to force update") + } + if len(del) != 0 { + return fmt.Errorf("going to delete arch(s) %s, you must use -f flag to force update", strings.Join(del, ",")) + } + } + + if !airgapFlags.isForce && utils.IsTerm() { + if versionChanged && + !utils.AskForConfirmation(fmt.Sprintf("New k3s version %s is summitted, old version package will be removed.", airgapFlags.K3sVersion), false) { + return nil + } + if len(del) != 0 && + !utils.AskForConfirmation(fmt.Sprintf("Are you going to delete arch(s) %s", strings.Join(del, ",")), false) { + return nil + } + } + if !versionChanged && len(add) == 0 && len(del) == 0 { + if toUpdate.State == common.PackageActive && !airgapFlags.isForce { + cmd.Println("package not changed") + return nil + } + } else { + toUpdate.Archs = airgapFlags.Archs + toUpdate.State = common.PackageOutOfSync + if versionChanged { + toUpdate.K3sVersion = airgapFlags.K3sVersion + } + if err := common.DefaultDB.SavePackage(toUpdate); err != nil { + return err + } + cmd.Printf("package %s of k3s version %s updated with arch(s) %s\n", name, toUpdate.K3sVersion, strings.Join(toUpdate.Archs, ",")) + } + + downloader, err := pkgairgap.NewDownloader(toUpdate) + if err != nil { + return errors.Wrapf(err, "failed to start download process for package %s", toUpdate.Name) + } + filepath, err := downloader.DownloadPackage() + if err != nil { + return errors.Wrapf(err, "failed to download package %s", toUpdate.Name) + } + + toUpdate.State = common.PackageActive + toUpdate.FilePath = filepath + if err := common.DefaultDB.SavePackage(toUpdate); err != nil { + return err + } + + cmd.Println("package updated") + return nil +} diff --git a/cmd/airgap/update_script.go b/cmd/airgap/update_script.go new file mode 100644 index 00000000..e56022d0 --- /dev/null +++ b/cmd/airgap/update_script.go @@ -0,0 +1,30 @@ +package airgap + +import ( + "bytes" + + "github.com/cnrancher/autok3s/pkg/settings" + + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +var ( + updateScriptCmd = &cobra.Command{ + Use: "update-install-script", + Short: "Will update the embed k3s install.sh script.", + RunE: updateInstallScript, + } +) + +func updateInstallScript(cmd *cobra.Command, args []string) error { + buff := bytes.NewBuffer([]byte{}) + if err := settings.GetScriptFromSource(buff); err != nil { + return err + } + if err := settings.InstallScript.Set(buff.String()); err != nil { + return errors.Wrap(err, "failed to update install script") + } + cmd.Printf("update install.sh script from source %s done\n", settings.ScriptUpdateSource.Get()) + return nil +} diff --git a/cmd/serve.go b/cmd/serve.go index 04d7c046..4f023de8 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -29,6 +29,7 @@ func init() { // ServeCommand serve command. func ServeCommand() *cobra.Command { serveCmd.Run = func(cmd *cobra.Command, args []string) { + common.IsCLI = false router := server.Start() // start kube-explorer for K3s clusters diff --git a/hack/make-rules/autok3s.sh b/hack/make-rules/autok3s.sh index 6f6a5eb3..204fe17c 100755 --- a/hack/make-rules/autok3s.sh +++ b/hack/make-rules/autok3s.sh @@ -40,6 +40,12 @@ function ui() { cd ${CURR_DIR} } +function go_generate() { + autok3s::log::info "running go generate..." + go generate -x + autok3s::log::info "go generate done" +} + function lint() { [[ "${1:-}" != "only" ]] && mod autok3s::log::info "linting autok3s..." @@ -57,6 +63,7 @@ function lint() { function build() { [[ "${1:-}" != "only" ]] && lint ui + go_generate autok3s::log::info "building autok3s(${GIT_VERSION},${GIT_COMMIT},${GIT_TREE_STATE},${BUILD_DATE})..." local version_flags=" diff --git a/main.go b/main.go index 6446dc3f..ab09236e 100644 --- a/main.go +++ b/main.go @@ -1,3 +1,4 @@ +//go:generate go run pkg/settings/script/main.go ./pkg/settings/install.sh package main import ( @@ -7,6 +8,7 @@ import ( "time" "github.com/cnrancher/autok3s/cmd" + "github.com/cnrancher/autok3s/cmd/airgap" "github.com/cnrancher/autok3s/pkg/cli/kubectl" "github.com/cnrancher/autok3s/pkg/common" "github.com/cnrancher/autok3s/pkg/metrics" @@ -40,7 +42,8 @@ func main() { rootCmd := cmd.Command() rootCmd.AddCommand(cmd.CompletionCommand(), cmd.VersionCommand(gitVersion, gitCommit, gitTreeState, buildDate), cmd.ListCommand(), cmd.CreateCommand(), cmd.JoinCommand(), cmd.KubectlCommand(), cmd.DeleteCommand(), - cmd.SSHCommand(), cmd.DescribeCommand(), cmd.ServeCommand(), cmd.ExplorerCommand(), cmd.UpgradeCommand(), cmd.TelemetryCommand()) + cmd.SSHCommand(), cmd.DescribeCommand(), cmd.ServeCommand(), cmd.ExplorerCommand(), cmd.UpgradeCommand(), + cmd.TelemetryCommand(), airgap.Command()) rootCmd.PersistentPreRun = func(c *cobra.Command, args []string) { common.InitLogger(logrus.StandardLogger()) diff --git a/pkg/airgap/archs.go b/pkg/airgap/archs.go new file mode 100644 index 00000000..6fdabe54 --- /dev/null +++ b/pkg/airgap/archs.go @@ -0,0 +1,42 @@ +package airgap + +import ( + "fmt" + "sort" +) + +func ValidateArchs(archs []string) error { + for _, arch := range archs { + if !ValidatedArch[arch] { + return fmt.Errorf("arch %s is not validated", arch) + } + } + return nil +} + +func GetValidatedArchs() []string { + var rtn []string + for arch := range ValidatedArch { + rtn = append(rtn, arch) + } + sort.Strings(rtn) + return rtn +} + +func GetArchDiff(current, target []string) (add, del []string) { + currentMap := map[string]bool{} + for _, arch := range current { + currentMap[arch] = true + } + for _, arch := range target { + if !currentMap[arch] { + add = append(add, arch) + } else { + delete(currentMap, arch) + } + } + for arch := range currentMap { + del = append(del, arch) + } + return +} diff --git a/pkg/airgap/download.go b/pkg/airgap/download.go new file mode 100644 index 00000000..5782de6e --- /dev/null +++ b/pkg/airgap/download.go @@ -0,0 +1,475 @@ +package airgap + +import ( + "bufio" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "github.com/cnrancher/autok3s/pkg/common" + "github.com/cnrancher/autok3s/pkg/settings" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + tmpDirName = ".tmp" + tmpSuffix = ".tmp" + doneFilename = ".done" + versionFilename = "version.json" + imageListFilename = "k3s-images.txt" + checksumBaseName = "sha256sum" + checksumExt = ".txt" + checksumFilename = checksumBaseName + checksumExt +) + +var ( + packagePath = filepath.Join(common.CfgPath, "package") + packageTmpBasePath = filepath.Join(packagePath, tmpDirName) + downloadSourceMap = map[string]string{ + "github": "https://github.com/k3s-io/k3s/releases/download", + "aliyunoss": "https://rancher-mirror.oss-cn-beijing.aliyuncs.com/k3s", + } + ErrVersionNotFound = errors.New("version not found") + + separator = regexp.MustCompile(" +") + ValidatedArch = map[string]bool{ + "arm64": true, + "amd64": true, + "arm": true, + "s390s": true, + } + resourceSuffixes = map[string][]string{ + "k3s": {""}, + "k3s-airgap-images": {".tar.gz", ".tar"}, + checksumBaseName: {checksumExt}, + } +) + +type version struct { + Version string + Archs []string +} + +func (v *version) diff(pkg common.Package) (toAdd, toDel []string) { + if v == nil { + toAdd = pkg.Archs + return + } + if v.Version != pkg.K3sVersion { + toAdd = pkg.Archs + toDel = v.Archs + return + } + return GetArchDiff(v.Archs, pkg.Archs) +} + +func NewDownloader(pkg common.Package) (*Downloader, error) { + d := &Downloader{ + pkg: pkg, + basePath: PackagePath(pkg.Name), + source: settings.PackageDownloadSource.Get(), + } + + versionPath := d.pkg.K3sVersion + if d.source == "aliyunoss" { + versionPath = strings.ReplaceAll(versionPath, "+", "-") + } + versionPath = url.QueryEscape(versionPath) + baseURL := downloadSourceMap[d.source] + d.sourceURL = fmt.Sprintf("%s/%s", baseURL, versionPath) + + d.logger = logrus.WithFields(logrus.Fields{ + "package": pkg.Name, + "version": pkg.K3sVersion, + }) + sort.Strings(d.pkg.Archs) + + return d, d.validateVersion() +} + +type Downloader struct { + source string + sourceURL string + basePath string + imageListContent []byte + pkg common.Package + logger logrus.FieldLogger +} + +func (d *Downloader) DownloadPackage() (string, error) { + version, err := versionAndBasePath(d.basePath) + if err != nil { + return "", err + } + + toAddArchs, toDelArchs := version.diff(d.pkg) + if len(toAddArchs) == 0 && + len(toDelArchs) == 0 && + isDone(d.basePath) { + d.logger.Info("the package %s is ready, skip downloading resources.", d.pkg.Name) + return d.basePath, nil + } + if err := d.writeVersion(); err != nil { + return "", err + } + + for _, arch := range toDelArchs { + d.logger.Infof("removing package arch %s", arch) + if err := os.RemoveAll(filepath.Join(d.basePath, arch)); err != nil { + return "", err + } + } + + for _, arch := range d.pkg.Archs { + if err := os.MkdirAll(filepath.Join(d.basePath, arch), 0755); err != nil { + return "", err + } + d.logger.Infof("download %s resources", arch) + if err := d.downloadArch(arch); err != nil { + return "", err + } + } + + if err := done(d.basePath); err != nil { + return "", err + } + + _, err = VerifyFiles(d.basePath) + + return d.basePath, err +} + +func (d *Downloader) downloadArch(arch string) error { + if err := d.checkArchExists(arch); err != nil { + return err + } + basePath := filepath.Join(d.basePath, arch) + if isDone(basePath) { + d.logger.Infof("arch %s has downloaded, skipped download process.", arch) + return nil + } + + archImageList := filepath.Join(basePath, imageListFilename) + if err := ioutil.WriteFile(archImageList, d.imageListContent, 0644); err != nil { + return err + } + + for basename, v := range resourceSuffixes { + suffixes := getSuffixMapWithArchs(arch, basename, v) + for origin, suffix := range suffixes { + localFileName := basename + origin + fullPath := filepath.Join(basePath, localFileName) + // The download process is using tmp file to download. The file will be considered downloaded if the exact file exists. + if _, err := os.Lstat(fullPath); err == nil { + d.logger.Infof("%s resource %s exists, skip downloading", arch, basename) + break + } + + // resourceName is the file name online + resourceName := basename + suffix + d.logger.Infof("downloading %s for %s", localFileName, arch) + if err := download(fullPath, d.getFileURL(resourceName)); err != nil { + d.logger.Warnf("failed to download resource %s for %s, skip this resource, %v", localFileName, arch, err) + continue + } + break + } + } + + if err := verifyArchFiles(arch, d.basePath); err != nil { + return err + } + + d.logger.Infof("all downloaded files are validated for %s", arch) + + if err := done(basePath); err != nil { + return err + } + + d.logger.Infof("k3s resource for %s downloaded.", arch) + return nil +} + +func (d *Downloader) getFileURL(Filename string) string { + return fmt.Sprintf("%s/%s", d.sourceURL, Filename) +} + +// validateVersion will download k3s-images.txt to check the version exists or not. +func (d *Downloader) validateVersion() error { + downloadURL := d.getFileURL(imageListFilename) + d.logger.Debugf("downloading images file from %s", downloadURL) + resp, err := http.Get(downloadURL) + if err != nil { + return errors.Wrapf(err, "failed to download image list of k3s version %s, this version may be not validated.", d.pkg.K3sVersion) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + content, _ := ioutil.ReadAll(resp.Body) + d.logger.Debugf("failed to download image list resource, status code %d, data %s", resp.StatusCode, string(content)) + return ErrVersionNotFound + } + + content, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + d.imageListContent = content + return nil +} + +// writeVersion will also remove .done file as we assume that creating/updating version is happening. +func (d *Downloader) writeVersion() error { + versionPath := filepath.Join(d.basePath, versionFilename) + versionJSON := versionContent(d.pkg) + _ = os.RemoveAll(versionPath) + _ = os.RemoveAll(getDonePath(d.basePath)) + d.logger.Info("generating version file") + return ioutil.WriteFile(versionPath, versionJSON, 0644) +} + +// checkArchExists will fire HEAD request to server and check response +func (d *Downloader) checkArchExists(arch string) error { + target := checksumBaseName + "-" + arch + checksumExt + resp, err := http.Head(d.getFileURL(target)) + if err != nil { + return err + } + defer resp.Body.Close() + if int(resp.StatusCode/100) != 2 { + return fmt.Errorf("%s may not exist", arch) + } + return nil +} + +func download(file, fromURL string) error { + resp, err := http.Get(fromURL) + if err != nil { + return err + } + defer resp.Body.Close() + if int(resp.StatusCode/100) != 2 { + content, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("failed to download resource %s, %s", fromURL, string(content)) + } + + tmpFile := file + ".tmp" + _ = os.RemoveAll(tmpFile) + + fp, err := os.Create(tmpFile) + if err != nil { + return err + } + defer fp.Close() + + if _, err := io.Copy(fp, resp.Body); err != nil { + return err + } + + return os.Rename(tmpFile, file) +} + +func versionContent(pkg common.Package) []byte { + version := version{ + Version: pkg.K3sVersion, + Archs: pkg.Archs, + } + data, _ := json.Marshal(version) + return data +} + +func versionAndBasePath(basePath string) (*version, error) { + rtn, err := os.Lstat(basePath) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + + if os.IsNotExist(err) { + if err := os.MkdirAll(basePath, 0755); err != nil { + return nil, err + } + return nil, nil + } + if !rtn.IsDir() { + return nil, fmt.Errorf("package path %s must be a directory", basePath) + } + + versionPath := filepath.Join(basePath, versionFilename) + data, err := ioutil.ReadFile(versionPath) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + v := version{} + if err := json.Unmarshal(data, &v); err != nil { + logrus.Warnf("failed to decode existing version json, assuming no version specified, %v", err) + v.Version = "" + } else { + if isDone(basePath) { + return &v, nil + } + } + contents, err := ioutil.ReadDir(basePath) + if err != nil { + return nil, err + } + for _, f := range contents { + if f.IsDir() && ValidatedArch[f.Name()] { + v.Archs = append(v.Archs, f.Name()) + } + } + return &v, nil +} + +func verifyArchFiles(arch, basePath string) error { + archBase := filepath.Join(basePath, arch) + checksumMap, err := getHashMapFromFile(filepath.Join(archBase, checksumFilename)) + if err != nil { + return errors.Wrapf(err, "failed to get file hash map for arch %s", arch) + } + + for basename, v := range resourceSuffixes { + if basename == checksumBaseName { + continue + } + checked := false + for origin, suffix := range getSuffixMapWithArchs(arch, basename, v) { + localFileName := basename + origin + resourceName := basename + suffix + + ok, err := checkFileHash(filepath.Join(archBase, localFileName), checksumMap[resourceName]) + if os.IsNotExist(err) { + continue + } + if !ok { + return fmt.Errorf("checksum for file %s/%s mismatch", arch, localFileName) + } + checked = true + break + } + if !checked { + return fmt.Errorf("resource %s for %s check fail", basename, arch) + } + } + return nil +} + +func VerifyFiles(basePath string) (*common.Package, error) { + version, err := versionAndBasePath(basePath) + if err != nil { + return nil, err + } + if version == nil { + return nil, errors.New("version.json is missing") + } + + for _, arch := range version.Archs { + archBase := filepath.Join(basePath, arch) + if !isDone(archBase) { + return nil, fmt.Errorf("%s resources aren't available", arch) + } + if err := verifyArchFiles(arch, basePath); err != nil { + return nil, err + } + } + + return &common.Package{ + Archs: version.Archs, + K3sVersion: version.Version, + }, nil +} + +func getSuffixMapWithArchs(arch, baseName string, suffixes []string) map[string]string { + rtn := make(map[string]string, len(suffixes)) + for _, suffix := range suffixes { + if baseName == "k3s" && arch == "amd64" { + rtn[suffix] = suffix + } else if baseName == "k3s" && arch == "arm" { + // arm binary suffix is armhf + rtn[suffix] = "-armhf" + suffix + } else { + rtn[suffix] = "-" + arch + suffix + } + } + return rtn +} + +func getExt(filename string) (string, string) { + name := filename + var ext, currentExt string + + for currentExt = filepath.Ext(name); currentExt != ""; currentExt = filepath.Ext(name) { + ext = currentExt + ext + name = strings.TrimSuffix(name, currentExt) + } + return name, ext +} + +func getHashMapFromFile(path string) (map[string]string, error) { + fp, err := os.Open(path) + if err != nil { + return nil, errors.Wrapf(err, "checksum file not found") + } + defer fp.Close() + checksumMap := map[string]string{} + reader := bufio.NewReader(fp) + for { + // checksum file should be small enough to ignore isPrefix options. + line, _, err := reader.ReadLine() + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + arr := separator.Split(string(line), 2) + checksumMap[filepath.Base(arr[1])] = arr[0] + } + return checksumMap, nil +} + +func checkFileHash(filepath, targetHash string) (bool, error) { + hasher := sha256.New() + fp, err := os.Open(filepath) + if err != nil { + return false, err + } + defer fp.Close() + if _, err := io.Copy(hasher, fp); err != nil { + return false, err + } + return fmt.Sprintf("%x", hasher.Sum(nil)) == targetHash, nil +} + +func isDone(basePath string) bool { + done, _ := os.Lstat(getDonePath(basePath)) + return done != nil +} + +func done(basePath string) error { + return ioutil.WriteFile(getDonePath(basePath), []byte{}, 0644) +} + +func getDonePath(basePath string) string { + return filepath.Join(basePath, doneFilename) +} + +func RemovePackage(name string) error { + return os.RemoveAll(PackagePath(name)) +} + +func TempDir(name string) string { + return filepath.Join(packageTmpBasePath, name) +} + +func PackagePath(name string) string { + return filepath.Join(packagePath, name) +} diff --git a/pkg/airgap/download_test.go b/pkg/airgap/download_test.go new file mode 100644 index 00000000..f286eb8b --- /dev/null +++ b/pkg/airgap/download_test.go @@ -0,0 +1,103 @@ +package airgap + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCheckHash(t *testing.T) { + f, err := ioutil.TempFile(".", "test-checksum-****") + if err != nil { + t.Fatal(err) + } + filename := f.Name() + defer func() { + os.RemoveAll(filename) + }() + + if _, err := f.WriteString("abcd\n"); err != nil { + t.Fatal(err) + } + f.Close() + targetHash := "fc4b5fd6816f75a7c81fc8eaa9499d6a299bd803397166e8c4cf9280b801d62c" + ok, err := checkFileHash(filename, targetHash) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("target Hash misatch") + } +} + +func TestDone(t *testing.T) { + basepath := "." + if isDone(basepath) { + t.Fatal("return done before executing doneFunc.") + } + if err := done(basepath); err != nil { + t.Fatal(err) + } + defer os.RemoveAll(getDonePath(basepath)) + if !isDone(basepath) { + t.Fatal("return not done after executed doneFunc.") + } +} + +func TestGetExt(t *testing.T) { + toTest := "abc.tar.gz" + targetName := "abc" + targetExt := ".tar.gz" + name, ext := getExt(toTest) + assert.Equal(t, targetName, name) + assert.Equal(t, targetExt, ext) + + // targetAMD64Map := map[string][]string{ + // "k3s": {""}, + // "k3s-airgap-images": {"-amd64.tar.gz", "-amd64.tar"}, + // checksumBaseName: {"-amd64.txt"}, + // } + // targetARM64Map := map[string][]string{ + // "k3s": {"-arm64"}, + // "k3s-airgap-images": {"-arm64.tar.gz", "-arm64.tar"}, + // checksumBaseName: {"-arm64.txt"}, + // } + // amd64Map := getResourceMapWithArch("amd64") + // arm64Map := getResourceMapWithArch("arm64") + // assert.Equal(t, targetAMD64Map, amd64Map) + // assert.Equal(t, targetARM64Map, arm64Map) +} + +func TestSuffixWithArch(t *testing.T) { + type testcase struct { + arch string + basename string + suffix []string + target map[string]string + } + for _, c := range []testcase{ + { + arch: "amd64", + basename: "k3s", + suffix: []string{""}, + target: map[string]string{"": ""}, + }, + { + arch: "arm64", + basename: "k3s", + suffix: []string{""}, + target: map[string]string{"": "-arm64"}, + }, + { + arch: "arm64", + basename: "k3s-airgap-images", + suffix: []string{".tar.gz", ".tar"}, + target: map[string]string{".tar.gz": "-arm64.tar.gz", ".tar": "-arm64.tar"}, + }, + } { + rtn := getSuffixMapWithArchs(c.arch, c.basename, c.suffix) + assert.Equal(t, c.target, rtn) + } +} diff --git a/pkg/airgap/tarfiles.go b/pkg/airgap/tarfiles.go new file mode 100644 index 00000000..f978dd0d --- /dev/null +++ b/pkg/airgap/tarfiles.go @@ -0,0 +1,164 @@ +package airgap + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" +) + +func SaveToTmp(path, name string) (string, error) { + rtn := TempDir(name) + if err := os.MkdirAll(rtn, 0755); err != nil { + return rtn, err + } + if _, err := os.Lstat(path); err != nil { + return "", err + } + + fp, err := os.Open(path) + if err != nil { + return "", err + } + defer fp.Close() + gzReader, err := gzip.NewReader(fp) + if err != nil { + return "", err + } + defer gzReader.Close() + tarReader := tar.NewReader(gzReader) + for { + tr, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + fullpath := filepath.Join(rtn, tr.Name) + switch tr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(fullpath, 0755); err != nil { + return "", err + } + case tar.TypeReg: + parent := filepath.Dir(fullpath) + if _, err := os.Lstat(parent); err != nil { + if err := os.MkdirAll(parent, 0755); err != nil { + return "", err + } + } + if err := func(header *tar.Header) error { + outFile, err := os.OpenFile(fullpath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, header.FileInfo().Mode()) + if err != nil { + return err + } + defer outFile.Close() + if _, err := io.Copy(outFile, tarReader); err != nil { + return err + } + return nil + }(tr); err != nil { + return "", err + } + default: + // ignore unknown type. + } + } + return rtn, nil +} + +func TarAndGzip(from, to string) error { + _, err := os.Lstat(to) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + return fmt.Errorf("file %s exists, stop exporting", to) + } + + // related path to real path + files := map[string]string{} + if err := filepath.Walk(from, func(path string, info fs.FileInfo, err error) error { + if from == path || info.IsDir() { + return nil + } + f, _ := filepath.Rel(from, path) + + files[f] = path + return nil + }); err != nil { + return err + } + toFile, err := os.Create(to) + if err != nil { + return err + } + defer toFile.Close() + return createArchive(files, toFile) +} + +func createArchive(files map[string]string, buf io.Writer) error { + // Create new Writers for gzip and tar + // These writers are chained. Writing to the tar writer will + // write to the gzip writer which in turn will write to + // the "buf" writer + gw := gzip.NewWriter(buf) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + + // Iterate over files and add them to the tar archive + for filename, filepath := range files { + err := addToArchive(tw, filename, filepath) + if err != nil { + return err + } + } + + return nil +} + +func addToArchive(tw *tar.Writer, filename, path string) error { + // Open the file which will be written into the archive + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + // Get FileInfo about our file providing file size, mode, etc. + info, err := file.Stat() + if err != nil { + return err + } + + // Create a tar Header from the FileInfo data + header, err := tar.FileInfoHeader(info, info.Name()) + if err != nil { + return err + } + + // Use full path as name (FileInfoHeader only takes the basename) + // If we don't do this the directory strucuture would + // not be preserved + // https://golang.org/src/archive/tar/common.go?#L626 + header.Name = filename + + // Write file header to the tar archive + err = tw.WriteHeader(header) + if err != nil { + return err + } + + // Copy file content to tar archive + _, err = io.Copy(tw, file) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/common/common.go b/pkg/common/common.go index bf7d2e2f..fb2f51b8 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -48,6 +48,8 @@ const ( ) var ( + // IsCLI means that you are not running serve command. + IsCLI = true // Debug used to enable log debug level. Debug = false // CfgPath default config file dir. diff --git a/pkg/common/db.go b/pkg/common/db.go index 21b16754..45790141 100644 --- a/pkg/common/db.go +++ b/pkg/common/db.go @@ -8,6 +8,7 @@ import ( "github.com/glebarez/sqlite" "gorm.io/gorm" + "gorm.io/gorm/logger" ) var ( @@ -126,7 +127,7 @@ func InitStorage(ctx context.Context) error { } setup(store.DB) - if err := store.DB.AutoMigrate(&ClusterState{}, &Template{}); err != nil { + if err := store.DB.AutoMigrate(&ClusterState{}, &Template{}, &Package{}); err != nil { return err } @@ -138,7 +139,11 @@ func InitStorage(ctx context.Context) error { // GetDB open and returns database. func GetDB() (*gorm.DB, error) { dataSource := GetDataSource() - return gorm.Open(sqlite.Open(dataSource), &gorm.Config{}) + config := &gorm.Config{} + if IsCLI && !Debug { + config.Logger = logger.Default.LogMode(logger.Silent) + } + return gorm.Open(sqlite.Open(dataSource), config) } func setup(db *gorm.DB) { diff --git a/pkg/common/metrics.go b/pkg/common/metrics.go index 6f027511..6a07483a 100644 --- a/pkg/common/metrics.go +++ b/pkg/common/metrics.go @@ -2,7 +2,6 @@ package common import ( "strconv" - "syscall" "github.com/cnrancher/autok3s/pkg/metrics" "github.com/cnrancher/autok3s/pkg/settings" @@ -12,7 +11,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "golang.org/x/term" ) func SetupPrometheusMetrics(version string) { @@ -79,7 +77,7 @@ func MetricsPrompt(cmd *cobra.Command) { cmd.Use == "explorer" { return } - if !term.IsTerminal(int(syscall.Stdin)) { + if !utils.IsTerm() { logrus.Debug("disable promoting telemetry in non-terminal environment") return } diff --git a/pkg/common/package.go b/pkg/common/package.go new file mode 100644 index 00000000..1337e801 --- /dev/null +++ b/pkg/common/package.go @@ -0,0 +1,59 @@ +package common + +import ( + "fmt" + + "github.com/cnrancher/autok3s/pkg/types" + + "gorm.io/gorm" +) + +type State string + +var ( + // PackageActive is the state after package downloaded + PackageActive State = "Active" + // PackageOutOfSync is the state when downloading package fails + PackageOutOfSync State = "OutOfSync" +) + +type Package struct { + Name string `json:"name,omitempty" gorm:"primaryKey;->;<-:create"` + K3sVersion string `json:"k3sVersion,omitempty"` + Archs types.StringArray `json:"archs,omitempty" gorm:"type:text"` + FilePath string `json:"filePath,omitempty"` + State State `json:"state,omitempty"` +} + +func (s *Store) ListPackages(name *string) ([]Package, error) { + var rtn []Package + var singleRtn Package + var rtnDB *gorm.DB + if name == nil { + rtnDB = s.DB.Find(&rtn) + } else { + rtnDB = s.DB.Model(Package{}).First(&singleRtn, "name = ?", *name) + } + if rtnDB.Error != nil { + return nil, rtnDB.Error + } + if name != nil { + rtn = append(rtn, singleRtn) + } + return rtn, nil +} + +func (s *Store) SavePackage(pkg Package) error { + return s.DB.Save(pkg).Error +} + +func (s *Store) DeletePackage(name string) error { + pkg, err := s.ListPackages(&name) + if err == gorm.ErrRecordNotFound { + return fmt.Errorf("package name %s not found", name) + } + if err != nil { + return err + } + return s.DB.Delete(&pkg[0]).Error +} diff --git a/pkg/common/package_test.go b/pkg/common/package_test.go new file mode 100644 index 00000000..805d0c79 --- /dev/null +++ b/pkg/common/package_test.go @@ -0,0 +1 @@ +package common diff --git a/pkg/settings/script/main.go b/pkg/settings/script/main.go new file mode 100644 index 00000000..ca52cc52 --- /dev/null +++ b/pkg/settings/script/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + + "github.com/cnrancher/autok3s/pkg/settings" + + "github.com/sirupsen/logrus" +) + +const ( + defaultFileName = "install.sh" +) + +func main() { + if len(os.Args) != 2 { + logrus.Fatal("target path should be specified") + } + targetPath := os.Args[1] + info, err := os.Lstat(targetPath) + if err != nil && !os.IsNotExist(err) { + logrus.Fatal(err) + } + if err == nil && info.IsDir() { + targetPath = filepath.Join(strings.TrimSuffix(targetPath, "/"), defaultFileName) + } + targetFile, err := os.OpenFile(targetPath, os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + logrus.Fatalf("failed to create script file, %v", err) + } + defer targetFile.Close() + + if err := settings.GetScriptFromSource(targetFile); err != nil { + logrus.Fatalf("failed to get script from source, %v", err) + } +} diff --git a/pkg/settings/script_prod.go b/pkg/settings/script_prod.go new file mode 100644 index 00000000..fca2f897 --- /dev/null +++ b/pkg/settings/script_prod.go @@ -0,0 +1,21 @@ +//go:build prod +// +build prod + +package settings + +import ( + "embed" +) + +//go:embed install.sh +var asserts embed.FS + +func init() { + data, err := asserts.ReadFile("install.sh") + if err != nil { + panic("install.sh should be included when compiling autok3s") + } + set := settings[InstallScript.Name] + set.Default = string(data) + settings[InstallScript.Name] = set +} diff --git a/pkg/settings/setting.go b/pkg/settings/setting.go index ff3406cd..cfdbfc43 100644 --- a/pkg/settings/setting.go +++ b/pkg/settings/setting.go @@ -1,5 +1,13 @@ package settings +import ( + "io" + "net/http" + "net/url" + + "github.com/pkg/errors" +) + type Setting struct { Name string Default string @@ -19,6 +27,10 @@ var ( WhitelistDomain = newSetting("whitelist-domain", "", "the domains or ips which allowed in autok3s UI proxy") EnableMetrics = newSetting("enable-metrics", "promote", "Should enable telemetry or not") InstallUUID = newSetting("install-uuid", "", "The autok3s instance unique install id") + + InstallScript = newSetting("install-script", "", "The k3s offline install script with base64 encode") + ScriptUpdateSource = newSetting("install-script-source-repo", "https://rancher-mirror.oss-cn-beijing.aliyuncs.com/k3s/k3s-install.sh", "The install script auto update source, github or aliyun oss") + PackageDownloadSource = newSetting("package-download-source", "github", "The airgap package download source, github and aliyunoss are validated.") ) func newSetting( @@ -44,14 +56,35 @@ func SetProvider(p Provider) error { func (s Setting) Get() string { if provider == nil { - return s.Default + return settings[s.Name].Default } return provider.Get(s.Name) } func (s Setting) Set(value string) error { if provider == nil { + setting := settings[s.Name] + setting.Default = value + settings[s.Name] = setting return nil } return provider.Set(s.Name, value) } + +func GetScriptFromSource(writer io.Writer) error { + sourceURL := ScriptUpdateSource.Get() + if _, err := url.Parse(sourceURL); err != nil { + return errors.Wrap(err, "install script source url is not validated.") + } + + resp, err := http.Get(sourceURL) + if err != nil { + return errors.Wrap(err, "failed to make request to install script source url.") + } + defer resp.Body.Close() + + if _, err := io.Copy(writer, resp.Body); err != nil { + return errors.Wrap(err, "failed to copy data to target writer") + } + return nil +} diff --git a/pkg/types/autok3s.go b/pkg/types/autok3s.go index 390f6220..783dab44 100644 --- a/pkg/types/autok3s.go +++ b/pkg/types/autok3s.go @@ -28,7 +28,7 @@ type Metadata struct { Worker string `json:"worker" yaml:"worker"` Token string `json:"token,omitempty" yaml:"token,omitempty"` IP string `json:"ip,omitempty" yaml:"ip,omitempty"` - TLSSans StringArray `json:"tls-sans,omitempty" yaml:"tls-sans,omitempty" gorm:"type:stringArray"` + TLSSans StringArray `json:"tls-sans,omitempty" yaml:"tls-sans,omitempty" gorm:"type:text"` ClusterCidr string `json:"cluster-cidr,omitempty" yaml:"cluster-cidr,omitempty"` MasterExtraArgs string `json:"master-extra-args,omitempty" yaml:"master-extra-args,omitempty"` WorkerExtraArgs string `json:"worker-extra-args,omitempty" yaml:"worker-extra-args,omitempty"` @@ -161,5 +161,14 @@ func (a StringArray) Value() (driver.Value, error) { // GormDataType returns gorm data type. func (a StringArray) GormDataType() string { - return "stringArray" + return "string" +} + +func (a StringArray) Contains(target string) bool { + for _, content := range a { + if target == content { + return true + } + } + return false } diff --git a/pkg/utils/util.go b/pkg/utils/util.go index 9800b8eb..84c3be61 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -11,12 +11,14 @@ import ( "os" "reflect" "strings" + "syscall" "text/template" "time" "github.com/AlecAivazis/survey/v2" "github.com/rancher/wrangler/pkg/schemas" "github.com/sirupsen/logrus" + "golang.org/x/term" "k8s.io/apimachinery/pkg/util/wait" ) @@ -178,3 +180,7 @@ func GenerateRand() int { mrand.Seed(time.Now().UnixNano()) return mrand.Intn(255) } + +func IsTerm() bool { + return term.IsTerminal(int(syscall.Stdin)) +}