feat: implement CRI image management and pre-pull on K8s upgrade

Fixes siderolabs#6391 Implement a set of APIs and commands to manage images in the CRI, and pre-pull images on Kubernetes upgrades. Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
schaurian · Jul 11, 2023 · 8017afb · 8017afb
1 parent 1c2f19b
commit 8017afb
Show file tree

Hide file tree

Showing 27 changed files with 2,835 additions and 595 deletions.
diff --git a/api/common/common.proto b/api/common/common.proto
@@ -88,6 +88,12 @@ enum ContainerDriver {
   CRI = 1;
 }
 
+enum ContainerdNamespace {
+  NS_UNKNOWN = 0;
+  NS_SYSTEM = 1;
+  NS_CRI = 2;
+}
+
 message URL {
   string full_path = 1;
 }

diff --git a/api/machine/machine.proto b/api/machine/machine.proto
@@ -105,6 +105,10 @@ service MachineService {
   rpc MetaWrite(MetaWriteRequest) returns (MetaWriteResponse);
   // MetaDelete deletes a META key.
   rpc MetaDelete(MetaDeleteRequest) returns (MetaDeleteResponse);
+  // ImageList lists images in the CRI.
+  rpc ImageList(ImageListRequest) returns (stream ImageListResponse);
+  // ImagePull pulls an image into the CRI.
+  rpc ImagePull(ImagePullRequest) returns (ImagePullResponse);
 }
 
 // rpc applyConfiguration
@@ -1316,3 +1320,31 @@ message MetaDelete {
 message MetaDeleteResponse {
   repeated MetaDelete messages = 1;
 }
+
+message ImageListRequest {
+  // Containerd namespace to use.
+  common.ContainerdNamespace namespace = 1;
+}
+
+message ImageListResponse {
+  common.Metadata metadata = 1;
+  string name = 2;
+  string digest = 3;
+  int64 size = 4;
+  google.protobuf.Timestamp created_at = 5;
+}
+
+message ImagePullRequest {
+  // Containerd namespace to use.
+  common.ContainerdNamespace namespace = 1;
+  // Image reference to pull.
+  string reference = 2;
+}
+
+message ImagePull {
+  common.Metadata metadata = 1;
+}
+
+message ImagePullResponse {
+  repeated ImagePull messages = 1;
+}
diff --git a/cmd/talosctl/cmd/talos/image.go b/cmd/talosctl/cmd/talos/image.go
@@ -0,0 +1,160 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package talos
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"text/tabwriter"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"github.com/spf13/cobra"
+
+	"github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers"
+	"github.com/siderolabs/talos/pkg/images"
+	"github.com/siderolabs/talos/pkg/machinery/api/common"
+	"github.com/siderolabs/talos/pkg/machinery/api/machine"
+	"github.com/siderolabs/talos/pkg/machinery/client"
+	"github.com/siderolabs/talos/pkg/machinery/config/container"
+	"github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1"
+)
+
+type imageCmdFlagsType struct {
+	namespace string
+}
+
+var imageCmdFlags imageCmdFlagsType
+
+func (flags imageCmdFlagsType) apiNamespace() (common.ContainerdNamespace, error) {
+	switch flags.namespace {
+	case "cri":
+		return common.ContainerdNamespace_NS_CRI, nil
+	case "system":
+		return common.ContainerdNamespace_NS_SYSTEM, nil
+	default:
+		return 0, fmt.Errorf("unsupported namespace %q", flags.namespace)
+	}
+}
+
+// imagesCmd represents the image command.
+var imageCmd = &cobra.Command{
+	Use:     "image",
+	Aliases: []string{},
+	Short:   "Manage CRI containter images",
+	Long:    ``,
+	Args:    cobra.NoArgs,
+}
+
+// imageListCmd represents the image list command.
+var imageListCmd = &cobra.Command{
+	Use:     "list",
+	Aliases: []string{"l", "ls"},
+	Short:   "List CRI images",
+	Long:    ``,
+	Args:    cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return WithClient(func(ctx context.Context, c *client.Client) error {
+			ns, err := imageCmdFlags.apiNamespace()
+			if err != nil {
+				return err
+			}
+
+			rcv, err := c.ImageList(ctx, ns)
+			if err != nil {
+				return fmt.Errorf("error listing images: %w", err)
+			}
+
+			w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
+			fmt.Fprintln(w, "NODE\tIMAGE\tDIGEST\tSIZE\tCREATED")
+
+			if err = helpers.ReadGRPCStream(rcv, func(msg *machine.ImageListResponse, node string, multipleNodes bool) error {
+				fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
+					node,
+					msg.Name,
+					msg.Digest,
+					humanize.Bytes(uint64(msg.Size)),
+					msg.CreatedAt.AsTime().Format(time.RFC3339),
+				)
+
+				return nil
+			}); err != nil {
+				return err
+			}
+
+			return w.Flush()
+		})
+	},
+}
+
+// imagePullCmd represents the image pull command.
+var imagePullCmd = &cobra.Command{
+	Use:     "pull",
+	Aliases: []string{"p"},
+	Short:   "Pull an image into CRI",
+	Long:    ``,
+	Args:    cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return WithClient(func(ctx context.Context, c *client.Client) error {
+			ns, err := imageCmdFlags.apiNamespace()
+			if err != nil {
+				return err
+			}
+
+			err = c.ImagePull(ctx, ns, args[0])
+			if err != nil {
+				return fmt.Errorf("error pulling image: %w", err)
+			}
+
+			return nil
+		})
+	},
+}
+
+// imageDefaultCmd represents the image default command.
+var imageDefaultCmd = &cobra.Command{
+	Use:   "default",
+	Short: "List the default images used by Talos",
+	Long:  ``,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		images := images.List(container.NewV1Alpha1(&v1alpha1.Config{
+			MachineConfig: &v1alpha1.MachineConfig{
+				MachineKubelet: &v1alpha1.KubeletConfig{},
+			},
+			ClusterConfig: &v1alpha1.ClusterConfig{
+				EtcdConfig:              &v1alpha1.EtcdConfig{},
+				APIServerConfig:         &v1alpha1.APIServerConfig{},
+				ControllerManagerConfig: &v1alpha1.ControllerManagerConfig{},
+				SchedulerConfig:         &v1alpha1.SchedulerConfig{},
+				CoreDNSConfig:           &v1alpha1.CoreDNS{},
+				ProxyConfig:             &v1alpha1.ProxyConfig{},
+			},
+		}))
+
+		fmt.Printf("%s\n", images.Flannel)
+		fmt.Printf("%s\n", images.FlannelCNI)
+		fmt.Printf("%s\n", images.CoreDNS)
+		fmt.Printf("%s\n", images.Etcd)
+		fmt.Printf("%s\n", images.KubeAPIServer)
+		fmt.Printf("%s\n", images.KubeControllerManager)
+		fmt.Printf("%s\n", images.KubeScheduler)
+		fmt.Printf("%s\n", images.KubeProxy)
+		fmt.Printf("%s\n", images.Kubelet)
+		fmt.Printf("%s\n", images.Installer)
+		fmt.Printf("%s\n", images.Pause)
+
+		return nil
+	},
+}
+
+func init() {
+	imageCmd.PersistentFlags().StringVar(&imageCmdFlags.namespace, "namespace", "cri", "namespace to use: `system` (etcd and kubelet images) or `cri` for all Kubernetes workloads")
+	addCommand(imageCmd)
+
+	imageCmd.AddCommand(imageDefaultCmd)
+	imageCmd.AddCommand(imageListCmd)
+	imageCmd.AddCommand(imagePullCmd)
+}
diff --git a/cmd/talosctl/cmd/talos/images.go b/cmd/talosctl/cmd/talos/images.go
@@ -5,49 +5,18 @@
 package talos
 
 import (
-	"fmt"
-
 	"github.com/spf13/cobra"
-
-	"github.com/siderolabs/talos/pkg/images"
-	"github.com/siderolabs/talos/pkg/machinery/config/container"
-	"github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1"
 )
 
-// imagesCmd represents the images command.
+// imagesCmd represents the (deprecated) images command.
+//
+// TODO: remove in Talos 1.6, add 'images' as an alias to talosctl image.
 var imagesCmd = &cobra.Command{
-	Use:   "images",
-	Short: "List the default images used by Talos",
-	Long:  ``,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		images := images.List(container.NewV1Alpha1(&v1alpha1.Config{
-			MachineConfig: &v1alpha1.MachineConfig{
-				MachineKubelet: &v1alpha1.KubeletConfig{},
-			},
-			ClusterConfig: &v1alpha1.ClusterConfig{
-				EtcdConfig:              &v1alpha1.EtcdConfig{},
-				APIServerConfig:         &v1alpha1.APIServerConfig{},
-				ControllerManagerConfig: &v1alpha1.ControllerManagerConfig{},
-				SchedulerConfig:         &v1alpha1.SchedulerConfig{},
-				CoreDNSConfig:           &v1alpha1.CoreDNS{},
-				ProxyConfig:             &v1alpha1.ProxyConfig{},
-			},
-		}))
-
-		fmt.Printf("%s\n", images.Flannel)
-		fmt.Printf("%s\n", images.FlannelCNI)
-		fmt.Printf("%s\n", images.CoreDNS)
-		fmt.Printf("%s\n", images.Etcd)
-		fmt.Printf("%s\n", images.KubeAPIServer)
-		fmt.Printf("%s\n", images.KubeControllerManager)
-		fmt.Printf("%s\n", images.KubeScheduler)
-		fmt.Printf("%s\n", images.KubeProxy)
-		fmt.Printf("%s\n", images.Kubelet)
-		fmt.Printf("%s\n", images.Installer)
-		fmt.Printf("%s\n", images.Pause)
-
-		return nil
-	},
+	Use:    "images",
+	Short:  "List the default images used by Talos",
+	Long:   ``,
+	Hidden: true,
+	RunE:   imageDefaultCmd.RunE,
 }
 
 func init() {

diff --git a/cmd/talosctl/cmd/talos/upgrade-k8s.go b/cmd/talosctl/cmd/talos/upgrade-k8s.go
@@ -41,6 +41,7 @@ func init() {
 	upgradeK8sCmd.Flags().StringVar(&upgradeK8sCmdFlags.ToVersion, "to", constants.DefaultKubernetesVersion, "the Kubernetes control plane version to upgrade to")
 	upgradeK8sCmd.Flags().StringVar(&upgradeOptions.ControlPlaneEndpoint, "endpoint", "", "the cluster control plane endpoint")
 	upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.DryRun, "dry-run", false, "skip the actual upgrade and show the upgrade plan instead")
+	upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.PrePullImages, "pre-pull-images", true, "pre-pull images before upgrade")
 	upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.UpgradeKubelet, "upgrade-kubelet", true, "upgrade kubelet service")
 	addCommand(upgradeK8sCmd)
 }

diff --git a/hack/release.toml b/hack/release.toml
@@ -119,6 +119,37 @@ systemDiskEncryption:
 ```
 gRPC API definitions and a simple reference implementation of the KMS server can be found in this
 [repository](https://github.com/siderolabs/kms-client/blob/main/cmd/kms-server/main.go).
+"""
+
+    [notes.talosctl-images]
+        title = "`talosctl images` Command"
+        description="""\
+The command `talosctl images` was renamed to `talosctl image default`.
+
+The backward-compatible alias is kept in Talos 1.5, but it will be dropped in Talos 1.6.
+"""
+
+    [notes.talosctl-image]
+        title = "`talosctl image` Command"
+        description="""\
+A new set of commands was introduced to manage container images in the CRI:
+
+* `talosctl image list` shows list of available images
+* `talosctl image pull` allows to pre-pull an image into the CRI
+
+Both new commands accept `--namespace` flag with two possible values:
+
+* `cri` (default): images managed by the CRI (Kubernetes workloads)
+* `system`: images managed by Talos (`etcd` and `kubelet`)
+```
+"""
+
+    [notes.upgrade-k8s]
+        title = "`talosctl upgrade-k8s` Image Pre-pulling"
+        description="""\
+The command `talosctl upgrade-k8s` now by default pre-pulls images for Kubernetes controlplane components
+and kubelet. This provides an early check for missing images, and minimizes downtime during Kubernetes
+rolling component update.
 """
 
 [make_deps]

diff --git a/internal/app/apid/main.go b/internal/app/apid/main.go
@@ -126,6 +126,7 @@ func apidMain() error {
 		"/machine.MachineService/Dmesg",
 		"/machine.MachineService/EtcdSnapshot",
 		"/machine.MachineService/Events",
+		"/machine.MachineService/ImageList",
 		"/machine.MachineService/Kubeconfig",
 		"/machine.MachineService/List",
 		"/machine.MachineService/Logs",